Skip to content

Commit

Permalink
[Testing] Python 3 conversion on Testing folder
Browse files Browse the repository at this point in the history
Seeing failures from calibration waterfall (and fixed by crrev.com/c/3199782). Adding more conversion on the other files in the same folder.

Bug: chromium:1252452
Change-Id: I58a25872c8d997f0714a769fbf0d1bd79d6fb8d4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/ /3202582
Commit-Queue: Wenbin Zhang <[email protected]>
Reviewed-by: John Chen <[email protected]>
Cr-Commit-Position: refs/heads/main@{#927976}
  • Loading branch information
Wenbin Zhang authored and Chromium LUCI CQ committed Oct 5, 2021
1 parent c044a7b commit c56b125
Show file tree
Hide file tree
Showing 22 changed files with 130 additions and 119 deletions.
2 changes: 0 additions & 2 deletions testing/buildbot/generate_buildbot_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 17,8 @@
import itertools
import json
import os
import re
import string
import sys
import traceback

import buildbot_json_magic_substitutions as magic_substitutions

Expand Down
22 changes: 9 additions & 13 deletions testing/buildbot/scripts/upload_test_result_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 14,16 @@
with the 'file' scheme) to google storage.
"""

from __future__ import print_function

import argparse
import collections
import copy
import itertools
import json
import hashlib
import os
import shutil
import subprocess
import sys
import tempfile
import urlparse
import uuid

root_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', '..'))
Expand Down Expand Up @@ -94,7 91,7 @@ def get_tests(test_trie):

def upload_directory_to_gs(local_path, bucket, gs_path, dry_run):
if dry_run:
print 'would have uploaded %s to %s' % (local_path, gs_path)
print('would have uploaded %s to %s' % (local_path, gs_path))
return

# -m does multithreaded uploads, which is needed because we upload multiple
Expand All @@ -107,9 104,8 @@ def hash_artifacts(tests, artifact_root):
hashed_artifacts = []
# Sort for testing consistency.
for test_obj in sorted(tests.values()):
for name, location in sorted(
test_obj.get('artifacts', {}).items(),
key=lambda pair: pair[0]):
for name, location in sorted(list(test_obj.get('artifacts', {}).items()),
key=lambda pair: pair[0]):
absolute_filepath = os.path.join(artifact_root, location)
file_digest = get_file_digest(absolute_filepath)
# Location is set to file digest because it's relative to the google
Expand Down Expand Up @@ -204,8 200,8 @@ def main():

type_info = data.get('artifact_type_info')
if not type_info:
print 'File %r did not have %r top level key. Not processing.' % (
args.test_result_file, 'artifact_type_info')
print('File %r did not have %r top level key. Not processing.' %
(args.test_result_file, 'artifact_type_info'))
return 1

new_data = upload_artifacts(
Expand All @@ -215,8 211,8 @@ def main():
json.dump(new_data, f)

if new_data and not args.quiet:
print json.dumps(
new_data, indent=2, separators=(',', ': '), sort_keys=True)
print(json.dumps(new_data, indent=2, separators=(',', ': '),
sort_keys=True))
return 0

if __name__ == '__main__':
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 3,8 @@
# found in the LICENSE file.
"""Tests for upload_test_result_artifacts."""

from __future__ import print_function

import json
import mock
import os
Expand Down Expand Up @@ -64,7 66,7 @@ def _loadTest(self, json_data, upload):

def loadTestEndToEndSimple(self):
test_data = self.makeTestJson(1, 10)
print self._loadTest(test_data, False)
print(self._loadTest(test_data, False))

def loadTestEndToEndManySmall(self):
test_data = self.makeTestJson(1000, 10)
Expand Down
21 changes: 12 additions & 9 deletions testing/chromoting/browser_tests_launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 4,9 @@


"""Utility script to launch browser-tests on the Chromoting bot."""

from __future__ import print_function

import argparse
import time

Expand Down Expand Up @@ -67,10 70,10 @@ def LaunchBTCommand(args, command):
# record instances where a test passed despite a JID mismatch.
if jids_used and host_jid.rstrip() not in jids_used:
host_jid_mismatch = True
print 'Host JID mismatch. JID in host log = %s.' % host_jid.rstrip()
print 'Host JIDs used by test:'
print('Host JID mismatch. JID in host log = %s.' % host_jid.rstrip())
print('Host JIDs used by test:')
for jid in jids_used:
print jid
print(jid)

if host_jid_mismatch:
# The JID for the remote-host did not match the JID that was used for this
Expand All @@ -83,7 86,7 @@ def LaunchBTCommand(args, command):
time.sleep(30)
continue
elif jids_used:
print 'JID used by test matched me2me host JID: %s' % host_jid
print('JID used by test matched me2me host JID: %s' % host_jid)
else:
# There wasn't a mismatch and no JIDs were returned. If no JIDs were
# returned, that means the test didn't use any JIDs, so there is nothing
Expand All @@ -102,9 105,9 @@ def LaunchBTCommand(args, command):
# and, because sometimes that line gets logged even if the test
# eventually passes, we'll also look for "(TIMED OUT)", before retrying.
if BROWSER_NOT_STARTED_ERROR in results and TIME_OUT_INDICATOR in results:
print 'Browser-instance not started (http://crbug/480025). Retrying.'
print('Browser-instance not started (http://crbug/480025). Retrying.')
else:
print 'Test failed for unknown reason. Retrying.'
print('Test failed for unknown reason. Retrying.')

retries = 1

Expand Down Expand Up @@ -156,9 159,9 @@ def main(args):
try:
host_logs = main(command_line_args)
if TEST_FAILURE:
print ' AT LEAST 1 TEST FAILED '
print FAILING_TESTS.rstrip('\n')
print ' '
print(' AT LEAST 1 TEST FAILED ')
print(FAILING_TESTS.rstrip('\n'))
print(' ')
raise Exception('At least one test failed.')
finally:
# Stop host and cleanup user-profile-dir.
Expand Down
11 changes: 7 additions & 4 deletions testing/chromoting/chromoting_test_driver_launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 2,11 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""Utility script to run chromoting test driver tests on the Chromoting bot."""

from __future__ import print_function

import argparse

from chromoting_test_utilities import GetJidFromHostLog
Expand Down Expand Up @@ -38,7 41,7 @@ def LaunchCTDCommand(args, command):

if not host_jid:
# Host-JID not found in log. Let's not attempt to run this test.
print 'Host-JID not found in log %s.' % host_log_file_names[-1]
print('Host-JID not found in log %s.' % host_log_file_names[-1])
return '[Command failed]: %s, %s' % (command, host_log_file_names)

retries = 0
Expand Down Expand Up @@ -123,9 126,9 @@ def main(args):
try:
failing_tests, host_logs = main(command_line_args)
if failing_tests:
print ' FAILED TESTS '
print failing_tests.rstrip('\n')
print ' '
print(' FAILED TESTS ')
print(failing_tests.rstrip('\n'))
print(' ')
raise Exception('At least one test failed.')
finally:
# Stop host and cleanup user-profile-dir.
Expand Down
18 changes: 10 additions & 8 deletions testing/chromoting/chromoting_test_utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 4,8 @@

"""Utility script to run tests on the Chromoting bot."""

from __future__ import print_function

import hashlib
import os
from os.path import expanduser
Expand Down Expand Up @@ -47,13 49,13 @@ def RunCommandInSubProcess(command):

cmd_line = [command]
try:
print 'Going to run:\n%s' % command
print('Going to run:\n%s' % command)
results = subprocess.check_output(cmd_line, stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError, e:
except subprocess.CalledProcessError as e:
results = e.output
finally:
print results
print(results)
return results


Expand Down Expand Up @@ -133,7 135,7 @@ def RestartMe2MeHost():
# Stop chromoting host.
RunCommandInSubProcess(CHROMOTING_HOST_PATH ' --stop')
# Start chromoting host.
print 'Starting chromoting host from %s' % CHROMOTING_HOST_PATH
print('Starting chromoting host from %s' % CHROMOTING_HOST_PATH)
results = RunCommandInSubProcess(CHROMOTING_HOST_PATH ' --start')

os.chdir(previous_directory)
Expand All @@ -148,7 150,7 @@ def RestartMe2MeHost():
if HOST_READY_INDICATOR not in results:
# Host start failed. Print out host-log. Don't run any tests.
with open(log_file, 'r') as f:
print f.read()
print(f.read())
raise HostOperationFailedException('Host restart failed.')

return log_file
Expand Down Expand Up @@ -194,9 196,9 @@ def PrintRunningProcesses():
processes = psutil.get_process_list()
processes = sorted(processes, key=lambda process: process.name)

print 'List of running processes:\n'
print('List of running processes:\n')
for process in processes:
print process.name
print(process.name)


def PrintHostLogContents(host_log_files=None):
Expand All @@ -206,7 208,7 @@ def PrintHostLogContents(host_log_files=None):
with open(log_file, 'r') as log:
host_log_contents = '\nHOST LOG %s\n CONTENTS:\n%s' % (
log_file, log.read())
print host_log_contents
print(host_log_contents)


def TestCaseSetup(args):
Expand Down
4 changes: 2 additions & 2 deletions testing/chromoting/download_test_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 49,8 @@ def main():
cp_cmd = ['gsutil.py', 'cp', line, output_file]
try:
subprocess.check_call(cp_cmd)
except subprocess.CalledProcessError, e:
print e.output
except subprocess.CalledProcessError as e:
print(e.output)
sys.exit(1)

if __name__ == '__main__':
Expand Down
20 changes: 11 additions & 9 deletions testing/clusterfuzz/common/fuzzy_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 13,16 @@
import textwrap

from . import utils
if sys.version_info.major == 3:
unichr = chr # pylint: disable=redefined-builtin


def FuzzyInt(n):
"""Returns an integer derived from the input by one of several mutations."""
int_sizes = [8, 16, 32, 64, 128]
mutations = [
lambda n: utils.UniformExpoInteger(0, sys.maxint.bit_length() 1),
lambda n: -utils.UniformExpoInteger(0, sys.maxint.bit_length()),
lambda n: utils.UniformExpoInteger(0, sys.maxsize.bit_length() 1),
lambda n: -utils.UniformExpoInteger(0, sys.maxsize.bit_length()),
lambda n: 2 ** random.choice(int_sizes) - 1,
lambda n: 2 ** random.choice(int_sizes),
lambda n: 0,
Expand Down Expand Up @@ -55,9 57,9 @@ def FuzzyString(s):
# If we're still here, apply a more generic mutation
mutations = [
lambda s: "".join(random.choice(string.printable) for i in
xrange(utils.UniformExpoInteger(0, 14))),
range(utils.UniformExpoInteger(0, 14))),
lambda s: "".join(unichr(random.randint(0, sys.maxunicode)) for i in
xrange(utils.UniformExpoInteger(0, 14))).encode("utf-8"),
range(utils.UniformExpoInteger(0, 14))).encode("utf-8"),
lambda s: os.urandom(utils.UniformExpoInteger(0, 14)),
lambda s: s * utils.UniformExpoInteger(1, 5),
lambda s: s "A" * utils.UniformExpoInteger(0, 14),
Expand Down Expand Up @@ -124,7 126,7 @@ def Overwrite(self, value, location=None, amount=None):
if amount is None:
amount = utils.RandomLowInteger(min(1, len(self)), len(self) - location)
if hasattr(value, "__call__"):
new_elements = (value() for i in xrange(amount))
new_elements = (value() for i in range(amount))
else:
new_elements = itertools.repeat(value, amount)
self[location:location amount] = new_elements
Expand All @@ -140,7 142,7 @@ def Insert(self, value, location=None, amount=None, max_exponent=14):
if amount is None:
amount = utils.UniformExpoInteger(0, max_exponent)
if hasattr(value, "__call__"):
new_elements = (value() for i in xrange(amount))
new_elements = (value() for i in range(amount))
else:
new_elements = itertools.repeat(value, amount)
self[location:location] = new_elements
Expand Down Expand Up @@ -171,7 173,7 @@ def RandomMutation(self, count=None, new_element=""):
]
if count is None:
count = utils.RandomLowInteger(1, 5, beta=3.0)
for _ in xrange(count):
for _ in range(count):
random.choice(mutations)()


Expand All @@ -185,7 187,7 @@ def FlipBits(self, num_bits=None):
"""Flip num_bits bits in the buffer at random."""
if num_bits is None:
num_bits = utils.RandomLowInteger(min(1, len(self)), len(self) * 8)
for bit in random.sample(xrange(len(self) * 8), num_bits):
for bit in random.sample(range(len(self) * 8), num_bits):
self[bit / 8] ^= 1 << (bit % 8)

def RandomMutation(self, count=None):
Expand All @@ -203,5 205,5 @@ def RandomMutation(self, count=None):
]
if count is None:
count = utils.RandomLowInteger(1, 5, beta=3.0)
for _ in xrange(count):
for _ in range(count):
utils.WeightedChoice(mutations)()
10 changes: 6 additions & 4 deletions testing/merge_scripts/results_merger.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 3,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import copy
import json
import sys
Expand Down Expand Up @@ -179,7 181,7 @@ def _merge_json_test_result_format(shard_results_list):
if result_json:
raise MergeException( # pragma: no cover (covered by
# results_merger_unittest).
'Unmergable values %s' % result_json.keys())
'Unmergable values %s' % list(result_json.keys()))

return merged_results

Expand All @@ -202,7 204,7 @@ def merge_tries(source, dest):
pending_nodes = [('', dest, source)]
while pending_nodes:
prefix, dest_node, curr_node = pending_nodes.pop()
for k, v in curr_node.iteritems():
for k, v in curr_node.items():
if k in dest_node:
if not isinstance(v, dict):
raise MergeException(
Expand Down Expand Up @@ -234,7 236,7 @@ def sum_dicts(source, dest):
This is intended for use as a merge_func parameter to merge_value.
"""
for k, v in source.iteritems():
for k, v in source.items():
dest.setdefault(k, 0)
dest[k] = v

Expand Down Expand Up @@ -274,7 276,7 @@ def main(files):
for f in files[1:]:
sys.stderr.write('Merging %s\n' % f)
result = merge_test_results([result, json.load(open(f))])
print json.dumps(result)
print(json.dumps(result))
return 0


Expand Down
Loading

0 comments on commit c56b125

Please sign in to comment.