Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gsheetsImport: reduce the number of API calls #526

Merged
merged 3 commits into from
Apr 27, 2020
Merged

gsheetsImport: reduce the number of API calls #526

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5581e17
reduce the number of API calls and allow importing mulitple test resu…
alanb-sony Apr 17, 2020
3b76b29
add option to insert new rows at the top of the sheets
alanb-sony Apr 27, 2020
a119324
Add some help text for command line args and trivial refactoring
garethsb Apr 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 101 additions & 44 deletions utilities/run-test-suites/gsheetsImport/resultsImporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import gspread
import json
import sys
import copy

from oauth2client.service_account import ServiceAccountCredentials

Expand All @@ -31,17 +32,69 @@
"Not Applicable"]


def gsheets_import(test_results, worksheet, filename, start_col=1):
def get_range(worksheet_data, start_row, start_col, end_row, end_col):
result = []
for row_no in range(start_row-1, end_row):
row = worksheet_data[row_no] if row_no < len(worksheet_data) else []
for col_no in range(start_col-1, end_col):
result.append(gspread.Cell(row_no + 1, col_no + 1, row[col_no] if col_no < len(row) else ""))
return result


def ranges_equal(first, second):
if len(first) != len(second):
return False
for i in range(len(first)):
if (first[i].row != second[i].row or
first[i].col != second[i].col or
first[i].value != second[i].value):
return False
return True


def insert_row(worksheet, data, row):
data_values = [{
"userEnteredValue": ({"formulaValue": x} if x.startswith("=") else {"stringValue": x})
} for x in data]
worksheet.spreadsheet.batch_update({
'requests': [{
'insertDimension': {
'range': {
'sheetId': worksheet.id,
'dimension': 'ROWS',
'startIndex': row,
'endIndex': row + 1
},
'inheritFromBefore': False
}
}, {
'updateCells': {
'start': {
'sheetId': worksheet.id,
'rowIndex': row,
'columnIndex': 0,
},
'rows': [{
"values": data_values
}],
'fields': 'userEnteredValue'
}
}]
})


def append_row(worksheet, data):
worksheet.append_rows([data],
value_input_option='USER_ENTERED',
insert_data_option='INSERT_ROWS',
table_range="A1")


def gsheets_import(test_results, worksheet, filename, start_col=1, insert=False):
"""Upload results data to spreadsheet"""

Copy link
Contributor

@garethsb garethsb Apr 20, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should be using worksheet.get_all_values('FORMULA') given we're using 'USER_ENTERED' later?

worksheet_data = worksheet.get_all_values()
populated_rows = len(worksheet_data)
if populated_rows == 0:
# Blank spreadsheet, row 1 will be for column titles
current_row = 2
else:
current_row = populated_rows+1

# Columns before start_col reserved for manually entered details
start_col = max(1, start_col)

Expand All @@ -61,57 +114,54 @@ def gsheets_import(test_results, worksheet, filename, start_col=1):
next_col = max(results_col, len(worksheet_data[0])+1)

# Test Names
start_cell_addr = gspread.utils.rowcol_to_a1(1, 1)
end_cell_addr = gspread.utils.rowcol_to_a1(1, next_col)
cell_list_names = worksheet.range("{}:{}".format(start_cell_addr, end_cell_addr))[:-1]
cell_list_names = get_range(worksheet_data, 1, 1, 1, next_col)
original_cell_list_names = copy.deepcopy(cell_list_names)

# Results
start_cell_addr = gspread.utils.rowcol_to_a1(current_row, 1)
end_cell_addr = gspread.utils.rowcol_to_a1(current_row, next_col)
cell_list_results = worksheet.range("{}:{}".format(start_cell_addr, end_cell_addr))[:-1]
cell_list_results = [""] * len(cell_list_names)

# Columns for Filename, URLs Tested, Timestamp, Test Suite
current_index = start_col-1 # list is 0-indexed whereas rows/cols are 1-indexed
cell_list_names[current_index].value = "Filename"
cell_list_results[current_index].value = filename
cell_list_results[current_index] = filename
current_index += 1
cell_list_names[current_index].value = "URLs Tested"
try:
urls_tested = []
for endpoint in test_results["endpoints"]:
urls_tested.append("{}:{} ({})".format(endpoint["host"], endpoint["port"], endpoint["version"]))
cell_list_results[current_index].value = ", ".join(urls_tested)
cell_list_results[current_index] = ", ".join(urls_tested)
except Exception:
print(" * WARNING: JSON file does not include endpoints")
cell_list_results[current_index].value = test_results["url"]
cell_list_results[current_index] = test_results["url"]
current_index += 1
cell_list_names[current_index].value = "Timestamp"
cell_list_results[current_index].value = (datetime.datetime.utcfromtimestamp(test_results["timestamp"])
.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z')
cell_list_results[current_index] = (datetime.datetime.utcfromtimestamp(test_results["timestamp"])
.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z')
current_index += 1
cell_list_names[current_index].value = "Test Suite"
try:
cell_list_results[current_index].value = "{} ({})".format(test_results["suite"],
test_results["config"]["VERSION"])
cell_list_results[current_index] = "{} ({})".format(test_results["suite"],
test_results["config"]["VERSION"])
except Exception:
print(" * WARNING: JSON file does not include test suite version")
cell_list_results[current_index].value = test_results["suite"]
cell_list_results[current_index] = test_results["suite"]

# Columns for counts of Tests and each Test Status
results_addr = "{}:{}".format(gspread.utils.rowcol_to_a1(current_row, results_col),
gspread.utils.rowcol_to_a1(current_row, 1)[1:])
result_col_name = gspread.utils.rowcol_to_a1(1, results_col)[0:-1]
results_addr = 'INDIRECT("{}"&ROW()&":"&ROW())'.format(result_col_name)

current_index += 1
cell_list_names[current_index].value = "Tests"
# count non-empty cells on rest of this row
cell_list_results[current_index].value = "=COUNTIF({}, \"?*\")".format(results_addr)
cell_list_results[current_index] = "=COUNTIF({}, \"?*\")".format(results_addr)
for state in TEST_STATES:
current_index += 1
cell_list_names[current_index].value = state
# count cells on the rest of this row that match this column's status
current_col_addr = gspread.utils.rowcol_to_a1(1, cell_list_names[current_index].col)
cell_list_results[current_index].value = "=COUNTIF({}, CONCAT({},\"*\"))" \
.format(results_addr, current_col_addr)
cell_list_results[current_index] = "=COUNTIF({}, CONCAT({},\"*\"))" \
.format(results_addr, current_col_addr)

# Columns for the Results
for result in test_results["results"]:
Expand All @@ -121,42 +171,49 @@ def gsheets_import(test_results, worksheet, filename, start_col=1):
col = next((cell.col for cell in cell_list_names if cell.value == result["name"]), None)
if col:
index = col-1 # list is 0-indexed whereas rows/cols are 1-indexed
cell_list_results[index].value = cell_contents
cell_list_results[index] = cell_contents
else:
# Test name not found, append column (since gspread doesn't make it easy to insert one)
col = cell_list_names[-1].col+1 # = cell_list_results[-1].col+1
cell_list_names.append(gspread.Cell(1, col, result["name"]))
cell_list_results.append(gspread.Cell(current_row, col, cell_contents))
cell_list_results.append(cell_contents)

worksheet.update_cells(cell_list_names)
# 'USER_ENTERED' allows formulae to be used
worksheet.update_cells(cell_list_results, value_input_option='USER_ENTERED')
if not ranges_equal(original_cell_list_names, cell_list_names):
worksheet.update_cells(cell_list_names)
if insert:
insert_row(worksheet, cell_list_results, 1)
else:
append_row(worksheet, cell_list_results)


def main():
parser = argparse.ArgumentParser()
parser.add_argument("--json", required=True)
parser.add_argument("--sheet", required=True)
parser.add_argument("--credentials", default="credentials.json")
parser.add_argument("--start_col", default="1", type=int)
parser.add_argument("--json", required=True, nargs="+", help="json test results filename(s) to import")
parser.add_argument("--sheet", required=True, help="spreadsheet url")
parser.add_argument("--credentials", default="credentials.json", help="credentials filename")
parser.add_argument("--start_col", default="1", type=int, help="reserve some columns for manually entered details")
parser.add_argument("--insert", action="store_true", help="insert new results at the top rather than the bottom")
args = parser.parse_args()

credentials = ServiceAccountCredentials.from_json_keyfile_name(args.credentials, SCOPES)
gcloud = gspread.authorize(credentials)

spreadsheet = gcloud.open_by_url(args.sheet)

with open(args.json) as json_file:
test_results = json.load(json_file)
worksheets = spreadsheet.worksheets()

try:
worksheet = spreadsheet.worksheet(test_results["suite"])
except gspread.exceptions.WorksheetNotFound:
print(" * ERROR: Worksheet {} not found".format(test_results["suite"]))
# could add_worksheet?
sys.exit(1)
for json_file_name in args.json:
with open(json_file_name) as json_file:
test_results = json.load(json_file)

try:
worksheet = next(x for x in worksheets if x.title == test_results["suite"])
except StopIteration:
print(" * ERROR: Worksheet {} not found".format(test_results["suite"]))
# could add_worksheet?
sys.exit(1)

gsheets_import(test_results, worksheet, args.json, args.start_col)
gsheets_import(test_results, worksheet, json_file_name, args.start_col, args.insert)


if __name__ == '__main__':
Expand Down