Commit 3b09a784 authored by Kamlesh Todai's avatar Kamlesh Todai
Browse files

Merge branch 'master' of community.opengroup.org:osdu/platform/testing

parents 6a0ee6bc 10381cba
Pipeline #70405 failed with stages
in 52 seconds
#!/usr/bin/env python3
# importing the libraries
import os
import re
import xlsxwriter
from bs4 import BeautifulSoup
import requests
import datetime
from pybadges import badge
#
# This script processes all the html files that are generated by using the Postman Newman (htmlextra)
# program to run the postman collections for "OSDU Platform Validation project."
# it expects the report files to follow the assumptions and rules mentioned in file
# OSDU Platform Validation CI_CD_Notes.md of "Platform Validation project"
# At present it is assuming 11 collections being run each of the four platforms
# AWS, AZURE, GCP, IBM
# AT present
# Number of collection can vary, but number of platforms is fixed.
# Also it expects to have the collection report for each platform.
# The Collection run date is extracted from the first collection report processed.
# Also environment file is extracted from the first collection processed for each platform
#
# In future work can be done to make the number of platforms also variable
# At present this restriction is there because
# of the way the writing of the excel file is implemented.
# Also the Columns for each platform are fixed
# =======================================================================================================
# Method to process html report file and summarize the results in Excel file
def processFile(fnm, platform, rowNo, colOffset, envFileNamesPopulated):
# print("fileName: " + fnm)
# print("Platform: " + platform)
# print("Row: ", rowNo)
# print("Total Rows: ", totRows)
# print("Column offset", colOffset)
#
# Open the html file for reading
html = open(fnm, 'r').read() # local html
bs = BeautifulSoup(html, features="html.parser") # parse the file
# Find the desired section of the file, extract the text and split to extract required information
mydivs = bs.findAll("div", {"class": "col-sm-12 mb-3"})
txt = mydivs[0].get_text()
fileInfo = txt.split("\n")
# Get file and environment information
collectionName = ""
enviornmentName = ""
count = 0
pattern = re.compile("Collection:")
# Parse and extract all the necessary values
for elm in fileInfo:
if pattern.search(elm):
if (count == 0):
collectionstr = elm.split(":")
collectionName = collectionstr[1]
pattern = re.compile("Environment:")
count += 1
elif (count == 1):
environmentstr = elm.split(":")
enviornmentName = environmentstr[1]
#
print(collectionName)
print(enviornmentName)
# Save/Collect environment file names (only once, not for each collection)
if (envFileNamesPopulated == False):
if (colOffset == 0):
env_file_names.append(enviornmentName)
elif (colOffset == 6):
env_file_names.append(enviornmentName)
elif (colOffset == 10):
env_file_names.append(enviornmentName)
elif (colOffset == 14):
env_file_names.append(enviornmentName)
else:
print("Something wrong while populating env file names")
#
# Get the summary of Total requests pass/fail, Total assertions pass/fail, Total tests skipped
mytab = bs.findAll("table", {"class": "table table-striped table-bordered"})
txt2 = mytab[0].get_text()
summaryInfo = txt2.split()
# Initialize the values
totalRequestsPassed = 0
totalRequestsFailed = 0
totalAssertionPassed = 0
totalAssertionFailed = 0
skippedTests = 0
count = 0
# Parse and extract all the necessary values
for info in summaryInfo:
if (count == 5):
totalRequestsPassed = info
elif (count == 6):
totalRequestsFailed = info
elif (count == 16):
totalAssertionPassed = info
elif (count == 17):
totalAssertionFailed = info
elif (count == 20):
skippedTests = info
count += 1
# Compute the pass rate (percentage of test that passed for the collection); Avoid ZeroDivisionError
if (float(totalAssertionPassed) > 0.0):
passRate = (100 * ((float(totalAssertionPassed) - float(totalAssertionFailed)) / float(totalAssertionPassed)))
# Add up the total numbers for each platform
if (platform not in env_total_assertions):
env_total_assertions[platform] = 0
env_passing_assertions[platform] = 0
# Despite the name, "totalAssertionPassed" is the number executed, not the number that passed
env_total_assertions[platform] += int(totalAssertionPassed)
env_passing_assertions[platform] += int(totalAssertionPassed) - int(totalAssertionFailed)
else:
passRate = 0.0
# print("totalRequestsPassed: " + totalRequestsPassed)
# print("totalRequestsFailed: " + totalRequestsFailed)
# print("totalAssertionPassed: " + totalAssertionPassed)
# print("totalAssertionFailed: " + totalAssertionFailed)
# print("skippedTests: " + skippedTests)
# print(passRate)
#
# save the information
records.append((collectionName, totalRequestsPassed, totalAssertionPassed, totalAssertionFailed, passRate))
curRow = len(records) - 1
# print(records[curRow])
# write the data to Excel file
for col_num, data in enumerate(records[curRow]):
l_colNum = col_num
if (col_num > 1) and (colOffset > 5):
l_colNum = col_num + colOffset - 2
if (l_colNum == 0):
worksheet.write(rowNo, l_colNum, data, collection_cell_format)
elif (l_colNum == 1):
worksheet.write(rowNo, l_colNum, int(data), request_cell_format)
elif (l_colNum in pctColList):
if (passRate >= 100.0):
worksheet.write(rowNo, l_colNum, data, greenPct_cell_format)
elif (passRate < 60.0):
worksheet.write(rowNo, l_colNum, data, redPct_cell_format)
else:
worksheet.write(rowNo, l_colNum, data, pct_cell_format)
else:
worksheet.write(rowNo, l_colNum, int(data), cell_format)
#
# Extract run Date Info only once from the first collection
if (len(collection_run_date) == 0):
mydivs3 = bs.findAll("div", {"class": "col-md-9 col-lg-12 main"})
txt3 = mydivs3[0].get_text()
dateInfo = txt3.split("\n")
collection_run_date.append(dateInfo[2])
#
# =======================================================================================================
# Start of Main
print("Starting to summarize all the html reports")
records = []
env_file_names = []
env_file_names_populated = False
env_total_assertions = {}
env_passing_assertions = {}
collection_run_date = []
#
fillColList = [5, 9, 13] # column list that are acting as separator between platforms
pctColList = [4, 8, 12, 16] # column list that hold passRate (percent) values
passColList = [2, 6, 10, 14] # column list that hold number of request pass values
failColList = [3, 7, 11, 15] # column list that hold number of request fail values
#
# Define the headers for the excel file
records.append(("", "", "", "AWS", "", "", "", "AZURE", "", "", "", "GCP", "", "", "", "IBM", ""))
# Note: Pass column name was changed to Executed. The variable names were not changed only column name was changed.
records.append(("Collection Name", "#of Requests", "Executed", "Fail", "PassRate", "", "Executed", "Fail", "PassRate",
"", "Executed", "Fail", "PassRate", "", "Executed", "Fail", "PassRate"))
# Create and initialize the excel file
workbook = xlsxwriter.Workbook('NewmanHtmlReportsSummary.xlsx')
worksheet = workbook.add_worksheet()
# set the column width
worksheet.set_column(0, 0, 40) # Column for Collection Name
worksheet.set_column(1, 1, 12) # Column for number of request
for iCol in passColList:
worksheet.set_column(iCol, iCol, 10) # column width for executed
# set variaous formats that will be used to wirte the data to the Excel file
cell_format = workbook.add_format({'align': 'center', 'bold': True})
#
summ_cell_format = workbook.add_format({'align': 'left', 'bold': True})
summ_right_cell_format = workbook.add_format({'align': 'right', 'bold': True})
#
request_cell_format = workbook.add_format({'align': 'center', 'bg_color': '#D8E4BC'})
#
collection_cell_format = workbook.add_format({'align': 'center', 'bold': True})
collection_cell_format.set_font_color('blue')
#
total_cell_format = workbook.add_format({'align': 'center', 'bold': True, 'bg_color': 'orange', 'font_color': 'blue'})
#
total_num_cell_format = workbook.add_format(
{'align': 'center', 'bold': True, 'bg_color': 'orange', 'font_color': 'blue'})
total_num_cell_format.set_num_format('0')
#
total_pct_cell_format = workbook.add_format(
{'align': 'center', 'bold': True, 'bg_color': 'orange', 'font_color': 'blue'})
total_pct_cell_format.set_num_format('0.00')
#
pct_cell_format = workbook.add_format({'align': 'center', 'bold': True})
pct_cell_format.set_num_format('0.00')
#
greenPct_cell_format = workbook.add_format({'align': 'center', 'bold': True})
greenPct_cell_format.set_font_color('green')
greenPct_cell_format.set_num_format('0.00')
#
redPct_cell_format = workbook.add_format({'align': 'center', 'bold': True})
redPct_cell_format.set_font_color('red')
redPct_cell_format.set_num_format('0.00')
#
head_cell_format = workbook.add_format({'align': 'center', 'bold': True, 'bg_color': '#78B0DE'})
head_cell_format.set_font_size(14)
#
head2_cell_format = workbook.add_format({'align': 'center', 'bold': True, 'bg_color': '#BBD7EF', 'font_color': 'black'})
head2_cell_format.set_font_size(12)
#
head2_green_cell_format = workbook.add_format(
{'align': 'center', 'bold': True, 'bg_color': '#BBD7EF', 'font_color': 'green'})
head2_green_cell_format.set_font_size(12)
head2_red_cell_format = workbook.add_format(
{'align': 'center', 'bold': True, 'bg_color': '#BBD7EF', 'font_color': 'red'})
head2_red_cell_format.set_font_size(12)
#
# Write the header records to excel file
rowNum = 0
col_offset = 0
for row_num in records:
for col_num, data in enumerate(records[rowNum]):
if (rowNum == 0):
worksheet.write(rowNum, col_num, data, head_cell_format)
else:
# print("rowNum: ", rowNum, " col_num: ", col_num)
if (rowNum == 1):
if (col_num in passColList):
worksheet.write(rowNum, col_num, data, head2_green_cell_format)
elif (col_num in failColList):
worksheet.write(rowNum, col_num, data, head2_red_cell_format)
else:
worksheet.write(rowNum, col_num, data, head2_cell_format)
rowNum += 1
# process all the reports to summarize the results
curPlatform = ""
localRowNo = rowNum
collName = []
for file in sorted(os.listdir(".")):
if (file.endswith(".html")): # process only html files
fileParts = file.split(".")
if (fileParts[0] == "index"):
print("Skip - " + file) # skip processing of index.html file
continue
else:
# print(file)
# Extract platform from the filename and set the start column offset value for the platform
# First platform (AWS) start column offset also accomodates number of request column
if (fileParts[0] != curPlatform):
#
localRowNo = rowNum
curPlatform = fileParts[0]
#
if (fileParts[0] == "aws"):
col_offset = 0
elif (fileParts[0] == "azure"):
col_offset = 6
elif (fileParts[0] == "gcp"):
col_offset = 10
elif (fileParts[0] == "ibm"):
col_offset = 14
else:
print("Invalid platform: " + fileParts[0])
continue
env_file_names_populated = False # platform changed, save environment file
else:
env_file_names_populated = True # want to save environment file only once for each platform
if (fileParts[0] == "aws"):
collName.append(fileParts[1])
# Process, extract and write the information to the Excel file
processFile(file, fileParts[0], localRowNo, col_offset,
env_file_names_populated) # process html report file
localRowNo += 1
print("")
# Summarize the results
print("Total number of collections: ", len(collName))
print("Total number of reports summarized: ", (len(records) - 2))
print("Total number of platforms", ((len(records) - 2) / len(collName)))
# Populate the Separator columns
fill_cell_format = workbook.add_format({'bg_color': '#FFE1E5'})
total_row_num = len(collName) + 2
colList = [5, 9, 13]
for i in range(2, total_row_num):
for j in range(len(colList)):
worksheet.write(i, colList[j], "", fill_cell_format)
# Write the summary data to the Excel file, Values for the Total row, Collection run and summary report date and list of environment files
total_cols = 17
rowIndx = total_row_num
rowIndxStr = str(rowIndx + 1)
sumStartIdx = 3
sumEndIdx = rowIndx
sumStartIdxStr = str(sumStartIdx)
sumEndIdxStr = str(sumEndIdx)
# popluate the arrays to sum the pass, fail and passRate values for each platform as well as total number of requests
# e.g. [ ['B14', '{=SUM(B3:B13)}'],...,[...]]
# passRate = (100 * ( (float(totalAssertionPassed) - float(totalAssertionFailed)) / float(totalAssertionPassed) ) )
# E14 = 100 * ( (C14-D14)/C14) )
# "E"+rowIndxStr, "{=(100.0 * ( (C"+rowIndxStr+"-D"+rowIndxStr+")/C"+rowIndxStr +") )}"
col_lables = [["B" + rowIndxStr, "{=SUM(B" + sumStartIdxStr + ":B" + sumEndIdxStr + ")}"],
["C" + rowIndxStr, "{=SUM(C" + sumStartIdxStr + ":C" + sumEndIdxStr + ")}"],
["D" + rowIndxStr, "{=SUM(D" + sumStartIdxStr + ":D" + sumEndIdxStr + ")}"],
["E" + rowIndxStr, "{=(100.0 * ( (C" + rowIndxStr + "-D" + rowIndxStr + ")/C" + rowIndxStr + ") )}"],
["G" + rowIndxStr, "{=SUM(G" + sumStartIdxStr + ":G" + sumEndIdxStr + ")}"],
["H" + rowIndxStr, "{=SUM(H" + sumStartIdxStr + ":H" + sumEndIdxStr + ")}"],
["I" + rowIndxStr, "{=(100.0 * ( (G" + rowIndxStr + "-H" + rowIndxStr + ")/G" + rowIndxStr + ") )}"],
["K" + rowIndxStr, "{=SUM(K" + sumStartIdxStr + ":K" + sumEndIdxStr + ")}"],
["L" + rowIndxStr, "{=SUM(L" + sumStartIdxStr + ":L" + sumEndIdxStr + ")}"],
["M" + rowIndxStr, "{=(100.0 * ( (K" + rowIndxStr + "-L" + rowIndxStr + ")/K" + rowIndxStr + ") )}"],
["O" + rowIndxStr, "{=SUM(O" + sumStartIdxStr + ":O" + sumEndIdxStr + ")}"],
["P" + rowIndxStr, "{=SUM(P" + sumStartIdxStr + ":P" + sumEndIdxStr + ")}"],
["Q" + rowIndxStr, "{=(100.0 * ( (O" + rowIndxStr + "-P" + rowIndxStr + ")/O" + rowIndxStr + ") )}"]]
col_label_indx = 0
# Populate/write the Total Row
for j in range(total_cols):
if (j == 0):
worksheet.write(rowIndx, j, "Total", total_cell_format)
elif (j in fillColList):
worksheet.write(total_row_num, j, "", total_cell_format)
# continue
else:
colIndx = col_lables[col_label_indx]
if (j in pctColList):
worksheet.write_formula(colIndx[0], colIndx[1], total_pct_cell_format)
else:
worksheet.write_formula(colIndx[0], colIndx[1], total_num_cell_format)
col_label_indx += 1
# Write the collection run date
date_time_obj = datetime.datetime.strptime(collection_run_date[0], '%A, %d %B %Y %H:%M:%S')
str1 = "Collection run date: " + date_time_obj.strftime('%m/%d/%Y, %H:%M:%S')
rowIndx += 2
worksheet.write(rowIndx, 0, str1, summ_cell_format)
# Write the summary report date
# str1 = "Summary report run date:\t " + (datetime.datetime.now().strftime('%A, %d %B %Y %H:%M:%S'))
str1 = "Summary report date: " + (datetime.datetime.now().strftime('%m/%d/%Y, %H:%M:%S'))
rowIndx += 1
worksheet.write(rowIndx, 0, str1, summ_cell_format)
# Wrtie the list of environment files
rowIndx += 1
worksheet.write(rowIndx, 0, "Environment Files:", summ_cell_format)
numOfEnvs = len(env_file_names)
rowIndx += 1
for i in range(numOfEnvs):
worksheet.write(rowIndx, 0, env_file_names[i], cell_format)
rowIndx += 1
# Close the Excel file
workbook.close()
# Output the test result badges, stylized to match GitLab pipeline badges
for platform in env_total_assertions:
passRate = 100.0 * env_passing_assertions[platform] / env_total_assertions[platform]
if env_passing_assertions[platform] == env_total_assertions[platform]:
badgeColor = '#4c1' # green
elif passRate >= 60.0:
badgeColor = '#dfb317' # yellow
else:
badgeColor = '#e05d44' # red
with open(platform + '.svg', 'w') as f:
f.write(badge(left_text=platform, right_text=('%.1f%%' % passRate), right_color=badgeColor));
# Log some summary data
print(env_file_names)
print(collection_run_date)
print("Summary Report Date: ", datetime.datetime.now())
print("Done")
exit()
# End
This diff is collapsed.
......@@ -84,7 +84,8 @@ with open('test-execution.gitlab-ci.yml', 'w') as pipelineConfig:
' - source venv/bin/activate',
' - pip3 install -r requirements.txt',
' script:',
' - cd public && ../Summarize_Newman_Html_Reports.py',
' - cd public',
' - python ../Summarize_Newman_Html_Reports.py',
' when: always',
''
]))
......
......@@ -8,3 +8,5 @@ soupsieve==2.0.1
urllib3==1.25.10
XlsxWriter==1.3.6
pybadges==2.2.1
openpyxl~=3.0.7
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment