diff --git a/CHANGES.txt b/CHANGES.txt index ac896199a9..78a85f1219 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -28,6 +28,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ruff: Handle F401 exclusions more granularly, remove per-file exclusions. - Implement type hints for Environment and environment utilities. - Deprecated Python 3.7 & 3.8 support. + - Deprecated implicit imports in `SCons.Util`. While they're still functional, they no longer + contribute to intellisense unless the correct package is imported. + - Split `SCons.Util.UtilTests.py` to individual packages where appropriate. - Modernized GitHub Actions test runner. This consolidates the logic of existing test environments into a single package. Adjusted AppVeyor to only run for targets unsupported by GitHub runners. diff --git a/RELEASE.txt b/RELEASE.txt index ac64c91e38..33d79c88ec 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -25,6 +25,8 @@ DEPRECATED FUNCTIONALITY - Deprecated Python 3.7 & 3.8 support. +- Deprecated implicit imports in `SCons.Util`. While they're still functional, they no longer + contribute to intellisense unless the correct package is imported. - Deprecated using signature-file-per-directory (aka old-style SConsign) via the SConsignFile(name=None) call. @@ -116,7 +118,7 @@ DEVELOPMENT - Docbook tests: improve skip message, more clearly indicate which test need actual installed system programs (add -live suffix). - Implement type hints for Environment and environment utilities. - +- Split `SCons.Util.UtilTests.py` to individual packages where appropriate. - MSVC: Added a host/target batch file configuration table for Visual Studio 2026. Visual Studio 2026 removed support for 32-bit arm targets. diff --git a/SCons/Util/UtilTests.py b/SCons/Util/UtilTests.py index 264cc69a8e..7b2c9a5418 100644 --- a/SCons/Util/UtilTests.py +++ b/SCons/Util/UtilTests.py @@ -23,33 +23,24 @@ from __future__ import annotations -import functools -import hashlib import io import os import subprocess import sys import unittest -import unittest.mock -import warnings -from collections import UserDict, UserList, UserString, namedtuple +from collections import UserDict from typing import Callable import TestCmd -import SCons.Errors -import SCons.compat from SCons.Util import ( - ALLOWED_HASH_FORMATS, - AddPathIfNotExists, - AppendPath, CLVar, LogicalLines, NodeList, - PrependPath, Proxy, Selector, WhereIs, + _wait_for_process_to_die_non_psutil, adjustixes, containsAll, containsAny, @@ -57,44 +48,23 @@ dictify, display, flatten, - get_env_bool, - get_environment_var, get_native_path, - get_os_env_bool, - hash_collect, - hash_signature, - is_Dict, - is_List, - is_String, - is_Tuple, print_tree, render_tree, - set_hash_format, silent_intern, splitext, - to_String, - to_bytes, - to_str, wait_for_process_to_die, - _wait_for_process_to_die_non_psutil, -) -from SCons.Util.envs import is_valid_construction_var -from SCons.Util.hashes import ( - _attempt_init_of_python_3_9_hash_object, - _attempt_get_hash_function, - _get_hash_object, - _set_allowed_viable_default_hashes, ) try: - import psutil + import psutil # noqa: F401 has_psutil = True except ImportError: has_psutil = False # These Util classes have no unit tests. Some don't make sense to test? -# DisplayEngine, Delegate, MethodWrapper, UniqueList, Unbuffered, Null, NullSeq +# DisplayEngine, Delegate, UniqueList, Unbuffered class OutBuffer: @@ -304,89 +274,6 @@ def get_children(node): finally: sys.stdout = save_stdout - def test_is_Dict(self) -> None: - assert is_Dict({}) - assert is_Dict(UserDict()) - try: - class mydict(dict): - pass - except TypeError: - pass - else: - assert is_Dict(mydict({})) - assert not is_Dict([]) - assert not is_Dict(()) - assert not is_Dict("") - - - def test_is_List(self) -> None: - assert is_List([]) - assert is_List(UserList()) - try: - class mylist(list): - pass - except TypeError: - pass - else: - assert is_List(mylist([])) - assert not is_List(()) - assert not is_List({}) - assert not is_List("") - - def test_is_String(self) -> None: - assert is_String("") - assert is_String(UserString('')) - try: - class mystr(str): - pass - except TypeError: - pass - else: - assert is_String(mystr('')) - assert not is_String({}) - assert not is_String([]) - assert not is_String(()) - - def test_is_Tuple(self) -> None: - assert is_Tuple(()) - try: - class mytuple(tuple): - pass - except TypeError: - pass - else: - assert is_Tuple(mytuple(())) - assert not is_Tuple([]) - assert not is_Tuple({}) - assert not is_Tuple("") - - def test_to_Bytes(self) -> None: - """ Test the to_Bytes method""" - self.assertEqual(to_bytes('Hello'), - bytearray('Hello', 'utf-8'), - "Check that to_bytes creates byte array when presented with non byte string.") - - def test_to_String(self) -> None: - """Test the to_String() method.""" - assert to_String(1) == "1", to_String(1) - assert to_String([1, 2, 3]) == str([1, 2, 3]), to_String([1, 2, 3]) - assert to_String("foo") == "foo", to_String("foo") - assert to_String(None) == 'None' - # test low level string converters too - assert to_str(None) == 'None' - assert to_bytes(None) == b'None' - - s1 = UserString('blah') - assert to_String(s1) == s1, s1 - assert to_String(s1) == 'blah', s1 - - class Derived(UserString): - pass - - s2 = Derived('foo') - assert to_String(s2) == s2, s2 - assert to_String(s2) == 'foo', s2 - def test_WhereIs(self) -> None: test = TestCmd.TestCmd(workdir='') @@ -468,18 +355,6 @@ def test_WhereIs(self) -> None: finally: os.environ['PATH'] = env_path - def test_get_env_var(self) -> None: - """Testing get_environment_var().""" - assert get_environment_var("$FOO") == "FOO", get_environment_var("$FOO") - assert get_environment_var("${BAR}") == "BAR", get_environment_var("${BAR}") - assert get_environment_var("$FOO_BAR1234") == "FOO_BAR1234", get_environment_var("$FOO_BAR1234") - assert get_environment_var("${BAR_FOO1234}") == "BAR_FOO1234", get_environment_var("${BAR_FOO1234}") - assert get_environment_var("${BAR}FOO") is None, get_environment_var("${BAR}FOO") - assert get_environment_var("$BAR ") is None, get_environment_var("$BAR ") - assert get_environment_var("FOO$BAR") is None, get_environment_var("FOO$BAR") - assert get_environment_var("$FOO[0]") is None, get_environment_var("$FOO[0]") - assert get_environment_var("${some('complex expression')}") is None, get_environment_var( - "${some('complex expression')}") def test_Proxy(self) -> None: """Test generic Proxy class.""" @@ -543,122 +418,6 @@ def test_get_native_path(self) -> None: except OSError: pass - def test_PrependPath(self) -> None: - """Test prepending to a path""" - # have to specify the pathsep when adding so it's cross-platform - # new duplicates existing - "moves to front" - with self.subTest(): - p1: list | str = r'C:\dir\num\one;C:\dir\num\two' - p1 = PrependPath(p1, r'C:\dir\num\two', sep=';') - p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') - self.assertEqual(p1, r'C:\dir\num\three;C:\dir\num\two;C:\dir\num\one') - - # ... except with delete_existing false - with self.subTest(): - p2: list | str = r'C:\dir\num\one;C:\dir\num\two' - p2 = PrependPath(p2, r'C:\dir\num\two', sep=';', delete_existing=False) - p2 = PrependPath(p2, r'C:\dir\num\three', sep=';', delete_existing=False) - self.assertEqual(p2, r'C:\dir\num\three;C:\dir\num\one;C:\dir\num\two') - - # only last one is kept if there are dupes in new - with self.subTest(): - p3: list | str = r'C:\dir\num\one' - p3 = PrependPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') - self.assertEqual(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one') - - # try prepending a Dir Node - with self.subTest(): - p4: list | str = r'C:\dir\num\one' - test = TestCmd.TestCmd(workdir='') - test.subdir('sub') - subdir = test.workpath('sub') - p4 = PrependPath(p4, subdir, sep=';') - self.assertEqual(p4, rf'{subdir};C:\dir\num\one') - - # try with initial list, adding string (result stays a list) - with self.subTest(): - p5: list = [r'C:\dir\num\one', r'C:\dir\num\two'] - p5 = PrependPath(p5, r'C:\dir\num\two', sep=';') - self.assertEqual(p5, [r'C:\dir\num\two', r'C:\dir\num\one']) - p5 = PrependPath(p5, r'C:\dir\num\three', sep=';') - self.assertEqual(p5, [r'C:\dir\num\three', r'C:\dir\num\two', r'C:\dir\num\one']) - - # try with initial string, adding list (result stays a string) - with self.subTest(): - p6: list | str = r'C:\dir\num\one;C:\dir\num\two' - p6 = PrependPath(p6, [r'C:\dir\num\two', r'C:\dir\num\three'], sep=';') - self.assertEqual(p6, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one') - - - def test_AppendPath(self) -> None: - """Test appending to a path.""" - # have to specify the pathsep when adding so it's cross-platform - # new duplicates existing - "moves to end" - with self.subTest(): - p1: list | str = r'C:\dir\num\one;C:\dir\num\two' - p1 = AppendPath(p1, r'C:\dir\num\two', sep=';') - p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') - self.assertEqual(p1, r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three') - - # ... except with delete_existing false - with self.subTest(): - p2: list | str = r'C:\dir\num\one;C:\dir\num\two' - p2 = AppendPath(p1, r'C:\dir\num\one', sep=';', delete_existing=False) - p2 = AppendPath(p1, r'C:\dir\num\three', sep=';') - self.assertEqual(p2, r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three') - - # only last one is kept if there are dupes in new - with self.subTest(): - p3: list | str = r'C:\dir\num\one' - p3 = AppendPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') - self.assertEqual(p3, r'C:\dir\num\one;C:\dir\num\three;C:\dir\num\two') - - # try appending a Dir Node - with self.subTest(): - p4: list | str = r'C:\dir\num\one' - test = TestCmd.TestCmd(workdir='') - test.subdir('sub') - subdir = test.workpath('sub') - p4 = AppendPath(p4, subdir, sep=';') - self.assertEqual(p4, rf'C:\dir\num\one;{subdir}') - - # try with initial list, adding string (result stays a list) - with self.subTest(): - p5: list = [r'C:\dir\num\one', r'C:\dir\num\two'] - p5 = AppendPath(p5, r'C:\dir\num\two', sep=';') - p5 = AppendPath(p5, r'C:\dir\num\three', sep=';') - self.assertEqual(p5, [r'C:\dir\num\one', r'C:\dir\num\two', r'C:\dir\num\three']) - - # try with initia string, adding list (result stays a string) - with self.subTest(): - p6: list | str = r'C:\dir\num\one;C:\dir\num\two' - p6 = AppendPath(p6, [r'C:\dir\num\two', r'C:\dir\num\three'], sep=';') - self.assertEqual(p6, r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three') - - def test_addPathIfNotExists(self) -> None: - """Test the AddPathIfNotExists() function""" - env_dict = {'FOO': os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), - 'BAR': os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), - 'BLAT': [os.path.normpath('/foo/bar'), - os.path.normpath('/baz/blat')]} - AddPathIfNotExists(env_dict, 'FOO', os.path.normpath('/foo/bar')) - AddPathIfNotExists(env_dict, 'BAR', os.path.normpath('/bar/foo')) - AddPathIfNotExists(env_dict, 'BAZ', os.path.normpath('/foo/baz')) - AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/blat')) - AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/foo')) - - assert env_dict['FOO'] == os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), env_dict['FOO'] - assert env_dict['BAR'] == os.path.normpath('/bar/foo') + os.pathsep + \ - os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), env_dict['BAR'] - assert env_dict['BAZ'] == os.path.normpath('/foo/baz'), env_dict['BAZ'] - assert env_dict['BLAT'] == [os.path.normpath('/baz/foo'), - os.path.normpath('/foo/bar'), - os.path.normpath('/baz/blat')], env_dict['BLAT'] - def test_CLVar(self) -> None: """Test the command-line construction variable class""" @@ -914,201 +673,6 @@ def _test_wait_for_process( wait_fn(p.pid) -class HashTestCase(unittest.TestCase): - - def test_collect(self) -> None: - """Test collecting a list of signatures into a new signature value - """ - for algorithm, expected in { - 'md5': ('698d51a19d8a121ce581499d7b701668', - '8980c988edc2c78cc43ccb718c06efd5', - '53fd88c84ff8a285eb6e0a687e55b8c7'), - 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', - '42eda1b5dcb3586bccfb1c69f22f923145271d97', - '2eb2f7be4e883ebe52034281d818c91e1cf16256'), - 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', - '25235f0fcab8767b7b5ac6568786fbc4f7d5d83468f0626bf07c3dbeed391a7a', - 'f8d3d0729bf2427e2e81007588356332e7e8c4133fae4bceb173b93f33411d17'), - }.items(): - # if the current platform does not support the algorithm we're looking at, - # skip the test steps for that algorithm, but display a warning to the user - if algorithm not in ALLOWED_HASH_FORMATS: - warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) - else: - hs = functools.partial(hash_signature, hash_format=algorithm) - s = list(map(hs, ('111', '222', '333'))) - - assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) - assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) - assert expected[2] == hash_collect(s, hash_format=algorithm) - - def test_MD5signature(self) -> None: - """Test generating a signature""" - for algorithm, expected in { - 'md5': ('698d51a19d8a121ce581499d7b701668', - 'bcbe3365e6ac95ea2c0343a2395834dd'), - 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', - '1c6637a8f2e1f75e06ff9984894d6bd16a3a36a9'), - 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', - '9b871512327c09ce91dd649b3f96a63b7408ef267c8cc5710114e629730cb61f'), - }.items(): - # if the current platform does not support the algorithm we're looking at, - # skip the test steps for that algorithm, but display a warning to the user - if algorithm not in ALLOWED_HASH_FORMATS: - warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) - else: - s = hash_signature('111', hash_format=algorithm) - assert expected[0] == s, s - - s = hash_signature('222', hash_format=algorithm) - assert expected[1] == s, s - -# this uses mocking out, which is platform specific, however, the FIPS -# behavior this is testing is also platform-specific, and only would be -# visible in hosts running Linux with the fips_mode kernel flag along -# with using OpenSSL. - -class FIPSHashTestCase(unittest.TestCase): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - - ############################### - # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 - self.fake_md5=lambda usedforsecurity=True: (usedforsecurity, 'md5') - self.fake_sha1=lambda usedforsecurity=True: (usedforsecurity, 'sha1') - self.fake_sha256=lambda usedforsecurity=True: (usedforsecurity, 'sha256') - ############################### - - ############################### - # hashlib mocks - md5Available = unittest.mock.Mock(md5=self.fake_md5) - del md5Available.sha1 - del md5Available.sha256 - self.md5Available=md5Available - - md5Default = unittest.mock.Mock(md5=self.fake_md5, sha1=self.fake_sha1) - del md5Default.sha256 - self.md5Default=md5Default - - sha1Default = unittest.mock.Mock(sha1=self.fake_sha1, sha256=self.fake_sha256) - del sha1Default.md5 - self.sha1Default=sha1Default - - sha256Default = unittest.mock.Mock(sha256=self.fake_sha256, **{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError}) - self.sha256Default=sha256Default - - all_throw = unittest.mock.Mock(**{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError, 'sha256.side_effect': ValueError}) - self.all_throw=all_throw - - no_algorithms = unittest.mock.Mock() - del no_algorithms.md5 - del no_algorithms.sha1 - del no_algorithms.sha256 - del no_algorithms.nonexist - self.no_algorithms=no_algorithms - - unsupported_algorithm = unittest.mock.Mock(unsupported=self.fake_sha256) - del unsupported_algorithm.md5 - del unsupported_algorithm.sha1 - del unsupported_algorithm.sha256 - del unsupported_algorithm.unsupported - self.unsupported_algorithm=unsupported_algorithm - ############################### - - ############################### - # system version mocks - VersionInfo = namedtuple('VersionInfo', 'major minor micro releaselevel serial') - v3_8 = VersionInfo(3, 8, 199, 'super-beta', 1337) - v3_9 = VersionInfo(3, 9, 0, 'alpha', 0) - v4_8 = VersionInfo(4, 8, 0, 'final', 0) - - self.sys_v3_8 = unittest.mock.Mock(version_info=v3_8) - self.sys_v3_9 = unittest.mock.Mock(version_info=v3_9) - self.sys_v4_8 = unittest.mock.Mock(version_info=v4_8) - ############################### - - def test_basic_failover_bad_hashlib_hash_init(self) -> None: - """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), - the hash init function returns None""" - assert _attempt_init_of_python_3_9_hash_object(None) is None - - def test_basic_failover_bad_hashlib_hash_get(self) -> None: - """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), - the hash get function returns None""" - assert _attempt_get_hash_function("nonexist", self.no_algorithms) is None - - def test_usedforsecurity_flag_behavior(self) -> None: - """Test usedforsecurity flag -> should be set to 'True' on older versions of python, and 'False' on Python >= 3.9""" - for version, expected in { - self.sys_v3_8: (True, 'md5'), - self.sys_v3_9: (False, 'md5'), - self.sys_v4_8: (False, 'md5'), - }.items(): - assert _attempt_init_of_python_3_9_hash_object(self.fake_md5, version) == expected - - def test_automatic_default_to_md5(self) -> None: - """Test automatic default to md5 even if sha1 available""" - for version, expected in { - self.sys_v3_8: (True, 'md5'), - self.sys_v3_9: (False, 'md5'), - self.sys_v4_8: (False, 'md5'), - }.items(): - _set_allowed_viable_default_hashes(self.md5Default, version) - set_hash_format(None, self.md5Default, version) - assert _get_hash_object(None, self.md5Default, version) == expected - - def test_automatic_default_to_sha256(self) -> None: - """Test automatic default to sha256 if other algorithms available but throw""" - for version, expected in { - self.sys_v3_8: (True, 'sha256'), - self.sys_v3_9: (False, 'sha256'), - self.sys_v4_8: (False, 'sha256'), - }.items(): - _set_allowed_viable_default_hashes(self.sha256Default, version) - set_hash_format(None, self.sha256Default, version) - assert _get_hash_object(None, self.sha256Default, version) == expected - - def test_automatic_default_to_sha1(self) -> None: - """Test automatic default to sha1 if md5 is missing from hashlib entirely""" - for version, expected in { - self.sys_v3_8: (True, 'sha1'), - self.sys_v3_9: (False, 'sha1'), - self.sys_v4_8: (False, 'sha1'), - }.items(): - _set_allowed_viable_default_hashes(self.sha1Default, version) - set_hash_format(None, self.sha1Default, version) - assert _get_hash_object(None, self.sha1Default, version) == expected - - def test_no_available_algorithms(self) -> None: - """expect exceptions on no available algorithms or when all algorithms throw""" - self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.no_algorithms) - self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.all_throw) - self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.unsupported_algorithm) - - def test_bad_algorithm_set_attempt(self) -> None: - """expect exceptions on user setting an unsupported algorithm selections, either by host or by SCons""" - - # nonexistant hash algorithm, not supported by SCons - _set_allowed_viable_default_hashes(self.md5Available) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'blah blah blah', hashlib_used=self.no_algorithms) - - # md5 is default-allowed, but in this case throws when we attempt to use it - _set_allowed_viable_default_hashes(self.md5Available) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) - - # user attempts to use an algorithm that isn't supported by their current system but is supported by SCons - _set_allowed_viable_default_hashes(self.sha1Default) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) - - # user attempts to use an algorithm that is supported by their current system but isn't supported by SCons - _set_allowed_viable_default_hashes(self.sha1Default) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'unsupported', hashlib_used=self.unsupported_algorithm) - - def tearDown(self) -> None: - """Return SCons back to the normal global state for the hashing functions.""" - _set_allowed_viable_default_hashes(hashlib, sys) - set_hash_format(None) - class NodeListTestCase(unittest.TestCase): def test_simple_attributes(self) -> None: @@ -1178,136 +742,5 @@ def test_dictionary_values(self) -> None: self.assertEqual(sorted(result), [1, 2, 3]) -class OsEnviron: - """Used to temporarily mock os.environ""" - - def __init__(self, environ) -> None: - self._environ = environ - - def start(self) -> None: - self._stored = os.environ - os.environ = self._environ - - def stop(self) -> None: - os.environ = self._stored - del self._stored - - def __enter__(self): - self.start() - return os.environ - - def __exit__(self, *args) -> None: - self.stop() - - -class get_env_boolTestCase(unittest.TestCase): - def test_missing(self) -> None: - env = {} - var = get_env_bool(env, 'FOO') - assert var is False, "var should be False, not %s" % repr(var) - env = {'FOO': '1'} - var = get_env_bool(env, 'BAR') - assert var is False, "var should be False, not %s" % repr(var) - - def test_true(self) -> None: - for arg in ['TRUE', 'True', 'true', - 'YES', 'Yes', 'yes', - 'Y', 'y', - 'ON', 'On', 'on', - '1', '20', '-1']: - env = {'FOO': arg} - var = get_env_bool(env, 'FOO') - assert var is True, 'var should be True, not %s' % repr(var) - - def test_false(self) -> None: - for arg in ['FALSE', 'False', 'false', - 'NO', 'No', 'no', - 'N', 'n', - 'OFF', 'Off', 'off', - '0']: - env = {'FOO': arg} - var = get_env_bool(env, 'FOO', True) - assert var is False, 'var should be True, not %s' % repr(var) - - def test_default(self) -> None: - env = {'FOO': 'other'} - var = get_env_bool(env, 'FOO', True) - assert var is True, 'var should be True, not %s' % repr(var) - var = get_env_bool(env, 'FOO', False) - assert var is False, 'var should be False, not %s' % repr(var) - - -class get_os_env_boolTestCase(unittest.TestCase): - def test_missing(self) -> None: - with OsEnviron({}): - var = get_os_env_bool('FOO') - assert var is False, "var should be False, not %s" % repr(var) - with OsEnviron({'FOO': '1'}): - var = get_os_env_bool('BAR') - assert var is False, "var should be False, not %s" % repr(var) - - def test_true(self) -> None: - for arg in ['TRUE', 'True', 'true', - 'YES', 'Yes', 'yes', - 'Y', 'y', - 'ON', 'On', 'on', - '1', '20', '-1']: - with OsEnviron({'FOO': arg}): - var = get_os_env_bool('FOO') - assert var is True, 'var should be True, not %s' % repr(var) - - def test_false(self) -> None: - for arg in ['FALSE', 'False', 'false', - 'NO', 'No', 'no', - 'N', 'n', - 'OFF', 'Off', 'off', - '0']: - with OsEnviron({'FOO': arg}): - var = get_os_env_bool('FOO', True) - assert var is False, 'var should be True, not %s' % repr(var) - - def test_default(self) -> None: - with OsEnviron({'FOO': 'other'}): - var = get_os_env_bool('FOO', True) - assert var is True, 'var should be True, not %s' % repr(var) - var = get_os_env_bool('FOO', False) - assert var is False, 'var should be False, not %s' % repr(var) - - -class EnvironmentVariableTestCase(unittest.TestCase): - - def test_is_valid_construction_var(self) -> None: - """Testing is_valid_construction_var()""" - r = is_valid_construction_var("_a") - assert r, r - r = is_valid_construction_var("z_") - assert r, r - r = is_valid_construction_var("X_") - assert r, r - r = is_valid_construction_var("2a") - assert not r, r - r = is_valid_construction_var("a2_") - assert r, r - r = is_valid_construction_var("/") - assert not r, r - r = is_valid_construction_var("_/") - assert not r, r - r = is_valid_construction_var("a/") - assert not r, r - r = is_valid_construction_var(".b") - assert not r, r - r = is_valid_construction_var("_.b") - assert not r, r - r = is_valid_construction_var("b1._") - assert not r, r - r = is_valid_construction_var("-b") - assert not r, r - r = is_valid_construction_var("_-b") - assert not r, r - r = is_valid_construction_var("b1-_") - assert not r, r - - - if __name__ == "__main__": unittest.main() diff --git a/SCons/Util/__init__.py b/SCons/Util/__init__.py index aca973bd6f..2c80c48ab0 100644 --- a/SCons/Util/__init__.py +++ b/SCons/Util/__init__.py @@ -59,57 +59,62 @@ import time from collections import UserDict, UserList, deque from contextlib import suppress -from typing import Any from logging import Formatter - -# Util split into a package. Make sure things that used to work -# when importing just Util itself still work: -from .sctypes import ( # noqa: F401 - DictTypes, - ListTypes, - SequenceTypes, - StringTypes, - BaseStringTypes, - Null, - NullSeq, - is_Dict, - is_List, - is_Sequence, - is_Tuple, - is_String, - is_Scalar, - to_String, - to_String_for_subst, - to_String_for_signature, - to_Text, - to_bytes, - to_str, - get_env_bool, - get_os_env_bool, - get_environment_var, -) -from .hashes import ( # noqa: F401 - ALLOWED_HASH_FORMATS, - DEFAULT_HASH_FORMATS, - get_hash_format, - set_hash_format, - get_current_hash_algorithm_used, - hash_signature, - hash_file_signature, - hash_collect, - MD5signature, - MD5filesignature, - MD5collect, -) -from .envs import ( # noqa: F401 - MethodWrapper, - PrependPath, - AppendPath, - AddPathIfNotExists, - AddMethod, - is_valid_construction_var, -) -from .filelock import FileLock, SConsLockFailure # noqa: F401 +from typing import TYPE_CHECKING, Any + +import SCons.Util.sctypes + +# HACK: Util split into a package. We want to gradually remove the implicit dependencies defined +# here, but we also want to ensure that anything currently importing just Util itself still works. +# As a compromise, we'll exclude these from a typed context. That way, while everything will still +# function as-is, IDEs and intellisense will throw a fit if attempting this legacy access. +if not TYPE_CHECKING: + from .envs import ( # noqa: F401 + AddMethod, + AddPathIfNotExists, + AppendPath, + MethodWrapper, + PrependPath, + is_valid_construction_var, + ) + from .filelock import FileLock, SConsLockFailure # noqa: F401 + from .hashes import ( # noqa: F401 + ALLOWED_HASH_FORMATS, + DEFAULT_HASH_FORMATS, + MD5collect, + MD5filesignature, + MD5signature, + get_current_hash_algorithm_used, + get_hash_format, + hash_collect, + hash_file_signature, + hash_signature, + set_hash_format, + ) + from .sctypes import ( # noqa: F401 + BaseStringTypes, + DictTypes, + ListTypes, + Null, + NullSeq, + SequenceTypes, + StringTypes, + get_env_bool, + get_environment_var, + get_os_env_bool, + is_Dict, + is_List, + is_Scalar, + is_Sequence, + is_String, + is_Tuple, + to_bytes, + to_str, + to_String, + to_String_for_signature, + to_String_for_subst, + to_Text, + ) PYPY = hasattr(sys, 'pypy_translation_info') @@ -452,8 +457,8 @@ def do_flatten( # pylint: disable=redefined-outer-name,redefined-builtin sequence, result, isinstance=isinstance, - StringTypes=StringTypes, - SequenceTypes=SequenceTypes, + StringTypes=StringTypes, # type: ignore + SequenceTypes=SequenceTypes, # type: ignore ) -> None: for item in sequence: if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): @@ -465,8 +470,8 @@ def do_flatten( # pylint: disable=redefined-outer-name,redefined-builtin def flatten( # pylint: disable=redefined-outer-name,redefined-builtin obj, isinstance=isinstance, - StringTypes=StringTypes, - SequenceTypes=SequenceTypes, + StringTypes=StringTypes, # type: ignore + SequenceTypes=SequenceTypes, # type: ignore do_flatten=do_flatten, ) -> list: """Flatten a sequence to a non-nested list. @@ -489,8 +494,8 @@ def flatten( # pylint: disable=redefined-outer-name,redefined-builtin def flatten_sequence( # pylint: disable=redefined-outer-name,redefined-builtin sequence, isinstance=isinstance, - StringTypes=StringTypes, - SequenceTypes=SequenceTypes, + StringTypes=StringTypes, # type: ignore + SequenceTypes=SequenceTypes, # type: ignore do_flatten=do_flatten, ) -> list: """Flatten a sequence to a non-nested list. @@ -700,14 +705,14 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> str | None: path = os.environ['PATH'] except KeyError: return None - if is_String(path): + if is_String(path): # type: ignore path = path.split(os.pathsep) if pathext is None: try: pathext = os.environ['PATHEXT'] except KeyError: pathext = '.COM;.EXE;.BAT;.CMD' - if is_String(pathext): + if is_String(pathext): # type: ignore pathext = pathext.split(os.pathsep) for ext in pathext: if ext.lower() == file[-len(ext):].lower(): @@ -715,7 +720,7 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> str | None: break if reject is None: reject = [] - if not is_List(reject) and not is_Tuple(reject): + if not is_List(reject) and not is_Tuple(reject): # type: ignore reject = [reject] for p in path: f = os.path.join(p, file) @@ -737,7 +742,7 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> str | None: path = os.environ['PATH'] except KeyError: return None - if is_String(path): + if is_String(path): # type: ignore path = path.split(os.pathsep) if pathext is None: pathext = ['.exe', '.cmd'] @@ -747,7 +752,7 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> str | None: break if reject is None: reject = [] - if not is_List(reject) and not is_Tuple(reject): + if not is_List(reject) and not is_Tuple(reject): # type: ignore reject = [reject] for p in path: f = os.path.join(p, file) @@ -771,11 +776,11 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> str | None: path = os.environ['PATH'] except KeyError: return None - if is_String(path): + if is_String(path): # type: ignore path = path.split(os.pathsep) if reject is None: reject = [] - if not is_List(reject) and not is_Tuple(reject): + if not is_List(reject) and not is_Tuple(reject): # type: ignore reject = [reject] for p in path: f = os.path.join(p, file) @@ -858,10 +863,10 @@ def Split(arg) -> list: >>> print(Split(["stringlist", " preserving ", " spaces "])) ['stringlist', ' preserving ', ' spaces '] """ - if is_List(arg) or is_Tuple(arg): + if is_List(arg) or is_Tuple(arg): # type: ignore return arg - if is_String(arg): + if is_String(arg): # type: ignore return arg.split() return [arg] @@ -1400,7 +1405,7 @@ def sanitize_shell_env(execution_env: dict) -> dict: # Ensure that the ENV values are all strings: new_env = {} for key, value in execution_env.items(): - if is_List(value): + if is_List(value): # type: ignore # If the value is a list, then we assume it is a path list, # because that's a pretty common list-like value to stick # in an environment variable: diff --git a/SCons/Util/envs.py b/SCons/Util/envs.py index 4ca75c391a..f6d0f4b502 100644 --- a/SCons/Util/envs.py +++ b/SCons/Util/envs.py @@ -17,7 +17,7 @@ from types import FunctionType, MethodType from typing import Any, cast, overload -from SCons.Util.sctypes import is_List, is_String, is_Tuple +import SCons.Util.sctypes @overload @@ -74,13 +74,13 @@ def PrependPath( orig = oldpath is_list = True paths = orig - if not is_List(orig) and not is_Tuple(orig): + if not SCons.Util.sctypes.is_List(orig) and not SCons.Util.sctypes.is_Tuple(orig): paths = cast(str, paths).split(sep) is_list = False - if is_String(newpath): + if SCons.Util.sctypes.is_String(newpath): newpaths = cast(str, newpath).split(sep) - elif is_List(newpath) or is_Tuple(newpath): + elif SCons.Util.sctypes.is_List(newpath) or SCons.Util.sctypes.is_Tuple(orig): newpaths = cast(list, newpath) else: newpaths = cast(list, [newpath]) # might be a Dir @@ -185,13 +185,13 @@ def AppendPath( orig = oldpath is_list = True paths = orig - if not is_List(orig) and not is_Tuple(orig): + if not SCons.Util.sctypes.is_List(orig) and not SCons.Util.sctypes.is_Tuple(orig): paths = cast(str, paths).split(sep) is_list = False - if is_String(newpath): + if SCons.Util.sctypes.is_String(newpath): newpaths = cast(str, newpath).split(sep) - elif is_List(newpath) or is_Tuple(newpath): + elif SCons.Util.sctypes.is_List(newpath) or SCons.Util.sctypes.is_Tuple(orig): newpaths = cast(list, newpath) else: newpaths = cast(list, [newpath]) # might be a Dir @@ -259,7 +259,7 @@ def AddPathIfNotExists( try: is_list = True paths = env_dict[key] - if not is_List(env_dict[key]): + if not SCons.Util.sctypes.is_List(env_dict[key]): paths = cast(str, paths).split(sep) is_list = False if os.path.normcase(path) not in list(map(os.path.normcase, paths)): diff --git a/SCons/Util/envsTests.py b/SCons/Util/envsTests.py new file mode 100644 index 0000000000..b28eaa185f --- /dev/null +++ b/SCons/Util/envsTests.py @@ -0,0 +1,192 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import annotations + +import os +import unittest + +import TestCmd + +from SCons.Util.envs import ( + AddPathIfNotExists, + AppendPath, + PrependPath, + is_valid_construction_var, +) + +# These envs classes have no unit tests. +# MethodWrapper + + +class TestEnvs(unittest.TestCase): + def test_PrependPath(self) -> None: + """Test prepending to a path""" + # have to specify the pathsep when adding so it's cross-platform + # new duplicates existing - "moves to front" + with self.subTest(): + p1: list | str = r'C:\dir\num\one;C:\dir\num\two' + p1 = PrependPath(p1, r'C:\dir\num\two', sep=';') + p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') + self.assertEqual(p1, r'C:\dir\num\three;C:\dir\num\two;C:\dir\num\one') + + # ... except with delete_existing false + with self.subTest(): + p2: list | str = r'C:\dir\num\one;C:\dir\num\two' + p2 = PrependPath(p2, r'C:\dir\num\two', sep=';', delete_existing=False) + p2 = PrependPath(p2, r'C:\dir\num\three', sep=';', delete_existing=False) + self.assertEqual(p2, r'C:\dir\num\three;C:\dir\num\one;C:\dir\num\two') + + # only last one is kept if there are dupes in new + with self.subTest(): + p3: list | str = r'C:\dir\num\one' + p3 = PrependPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') + self.assertEqual(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one') + + # try prepending a Dir Node + with self.subTest(): + p4: list | str = r'C:\dir\num\one' + test = TestCmd.TestCmd(workdir='') + test.subdir('sub') + subdir = test.workpath('sub') + p4 = PrependPath(p4, subdir, sep=';') + self.assertEqual(p4, rf'{subdir};C:\dir\num\one') + + # try with initial list, adding string (result stays a list) + with self.subTest(): + p5: list = [r'C:\dir\num\one', r'C:\dir\num\two'] + p5 = PrependPath(p5, r'C:\dir\num\two', sep=';') + self.assertEqual(p5, [r'C:\dir\num\two', r'C:\dir\num\one']) + p5 = PrependPath(p5, r'C:\dir\num\three', sep=';') + self.assertEqual(p5, [r'C:\dir\num\three', r'C:\dir\num\two', r'C:\dir\num\one']) + + # try with initial string, adding list (result stays a string) + with self.subTest(): + p6: list | str = r'C:\dir\num\one;C:\dir\num\two' + p6 = PrependPath(p6, [r'C:\dir\num\two', r'C:\dir\num\three'], sep=';') + self.assertEqual(p6, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one') + + + def test_AppendPath(self) -> None: + """Test appending to a path.""" + # have to specify the pathsep when adding so it's cross-platform + # new duplicates existing - "moves to end" + with self.subTest(): + p1: list | str = r'C:\dir\num\one;C:\dir\num\two' + p1 = AppendPath(p1, r'C:\dir\num\two', sep=';') + p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') + self.assertEqual(p1, r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three') + + # ... except with delete_existing false + with self.subTest(): + p2: list | str = r'C:\dir\num\one;C:\dir\num\two' + p2 = AppendPath(p1, r'C:\dir\num\one', sep=';', delete_existing=False) + p2 = AppendPath(p1, r'C:\dir\num\three', sep=';') + self.assertEqual(p2, r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three') + + # only last one is kept if there are dupes in new + with self.subTest(): + p3: list | str = r'C:\dir\num\one' + p3 = AppendPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') + self.assertEqual(p3, r'C:\dir\num\one;C:\dir\num\three;C:\dir\num\two') + + # try appending a Dir Node + with self.subTest(): + p4: list | str = r'C:\dir\num\one' + test = TestCmd.TestCmd(workdir='') + test.subdir('sub') + subdir = test.workpath('sub') + p4 = AppendPath(p4, subdir, sep=';') + self.assertEqual(p4, rf'C:\dir\num\one;{subdir}') + + # try with initial list, adding string (result stays a list) + with self.subTest(): + p5: list = [r'C:\dir\num\one', r'C:\dir\num\two'] + p5 = AppendPath(p5, r'C:\dir\num\two', sep=';') + p5 = AppendPath(p5, r'C:\dir\num\three', sep=';') + self.assertEqual(p5, [r'C:\dir\num\one', r'C:\dir\num\two', r'C:\dir\num\three']) + + # try with initia string, adding list (result stays a string) + with self.subTest(): + p6: list | str = r'C:\dir\num\one;C:\dir\num\two' + p6 = AppendPath(p6, [r'C:\dir\num\two', r'C:\dir\num\three'], sep=';') + self.assertEqual(p6, r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three') + + def test_addPathIfNotExists(self) -> None: + """Test the AddPathIfNotExists() function""" + env_dict = {'FOO': os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), + 'BAR': os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), + 'BLAT': [os.path.normpath('/foo/bar'), + os.path.normpath('/baz/blat')]} + AddPathIfNotExists(env_dict, 'FOO', os.path.normpath('/foo/bar')) + AddPathIfNotExists(env_dict, 'BAR', os.path.normpath('/bar/foo')) + AddPathIfNotExists(env_dict, 'BAZ', os.path.normpath('/foo/baz')) + AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/blat')) + AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/foo')) + + assert env_dict['FOO'] == os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), env_dict['FOO'] + assert env_dict['BAR'] == os.path.normpath('/bar/foo') + os.pathsep + \ + os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), env_dict['BAR'] + assert env_dict['BAZ'] == os.path.normpath('/foo/baz'), env_dict['BAZ'] + assert env_dict['BLAT'] == [os.path.normpath('/baz/foo'), + os.path.normpath('/foo/bar'), + os.path.normpath('/baz/blat')], env_dict['BLAT'] + + def test_is_valid_construction_var(self) -> None: + """Testing is_valid_construction_var()""" + r = is_valid_construction_var("_a") + assert r, r + r = is_valid_construction_var("z_") + assert r, r + r = is_valid_construction_var("X_") + assert r, r + r = is_valid_construction_var("2a") + assert not r, r + r = is_valid_construction_var("a2_") + assert r, r + r = is_valid_construction_var("/") + assert not r, r + r = is_valid_construction_var("_/") + assert not r, r + r = is_valid_construction_var("a/") + assert not r, r + r = is_valid_construction_var(".b") + assert not r, r + r = is_valid_construction_var("_.b") + assert not r, r + r = is_valid_construction_var("b1._") + assert not r, r + r = is_valid_construction_var("-b") + assert not r, r + r = is_valid_construction_var("_-b") + assert not r, r + r = is_valid_construction_var("b1-_") + assert not r, r + + +if __name__ == "__main__": + unittest.main() diff --git a/SCons/Util/hashes.py b/SCons/Util/hashes.py index 4950f6805e..3cdd5b5cf9 100644 --- a/SCons/Util/hashes.py +++ b/SCons/Util/hashes.py @@ -15,7 +15,6 @@ from .sctypes import to_bytes - # Default hash function and format. SCons-internal. DEFAULT_HASH_FORMATS = ['md5', 'sha1', 'sha256'] ALLOWED_HASH_FORMATS = [] diff --git a/SCons/Util/hashesTests.py b/SCons/Util/hashesTests.py new file mode 100644 index 0000000000..54be1d5234 --- /dev/null +++ b/SCons/Util/hashesTests.py @@ -0,0 +1,243 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import annotations + +import functools +import hashlib +import sys +import unittest +import unittest.mock +import warnings +from collections import namedtuple + +import SCons.Errors +from SCons.Util.hashes import ( + ALLOWED_HASH_FORMATS, + _attempt_get_hash_function, + _attempt_init_of_python_3_9_hash_object, + _get_hash_object, + _set_allowed_viable_default_hashes, + hash_collect, + hash_signature, + set_hash_format, +) + + +class HashTestCase(unittest.TestCase): + def test_collect(self) -> None: + """Test collecting a list of signatures into a new signature value""" + for algorithm, expected in { + 'md5': ('698d51a19d8a121ce581499d7b701668', + '8980c988edc2c78cc43ccb718c06efd5', + '53fd88c84ff8a285eb6e0a687e55b8c7'), + 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', + '42eda1b5dcb3586bccfb1c69f22f923145271d97', + '2eb2f7be4e883ebe52034281d818c91e1cf16256'), + 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', + '25235f0fcab8767b7b5ac6568786fbc4f7d5d83468f0626bf07c3dbeed391a7a', + 'f8d3d0729bf2427e2e81007588356332e7e8c4133fae4bceb173b93f33411d17'), + }.items(): + # if the current platform does not support the algorithm we're looking at, + # skip the test steps for that algorithm, but display a warning to the user + if algorithm not in ALLOWED_HASH_FORMATS: + warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) + else: + hs = functools.partial(hash_signature, hash_format=algorithm) + s = list(map(hs, ('111', '222', '333'))) + + assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) + assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) + assert expected[2] == hash_collect(s, hash_format=algorithm) + + def test_MD5signature(self) -> None: + """Test generating a signature""" + for algorithm, expected in { + 'md5': ('698d51a19d8a121ce581499d7b701668', + 'bcbe3365e6ac95ea2c0343a2395834dd'), + 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', + '1c6637a8f2e1f75e06ff9984894d6bd16a3a36a9'), + 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', + '9b871512327c09ce91dd649b3f96a63b7408ef267c8cc5710114e629730cb61f'), + }.items(): + # if the current platform does not support the algorithm we're looking at, + # skip the test steps for that algorithm, but display a warning to the user + if algorithm not in ALLOWED_HASH_FORMATS: + warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) + else: + s = hash_signature('111', hash_format=algorithm) + assert expected[0] == s, s + + s = hash_signature('222', hash_format=algorithm) + assert expected[1] == s, s + + +# This uses mocking out, which is platform specific. However, the FIPS +# behavior this is testing is also platform-specific, and only would be +# visible in hosts running Linux with the `fips_mode` kernel flag along +# with using OpenSSL. + +class FIPSHashTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + ############################### + # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 + self.fake_md5=lambda usedforsecurity=True: (usedforsecurity, 'md5') + self.fake_sha1=lambda usedforsecurity=True: (usedforsecurity, 'sha1') + self.fake_sha256=lambda usedforsecurity=True: (usedforsecurity, 'sha256') + ############################### + + ############################### + # hashlib mocks + md5Available = unittest.mock.Mock(md5=self.fake_md5) + del md5Available.sha1 + del md5Available.sha256 + self.md5Available=md5Available + + md5Default = unittest.mock.Mock(md5=self.fake_md5, sha1=self.fake_sha1) + del md5Default.sha256 + self.md5Default=md5Default + + sha1Default = unittest.mock.Mock(sha1=self.fake_sha1, sha256=self.fake_sha256) + del sha1Default.md5 + self.sha1Default=sha1Default + + sha256Default = unittest.mock.Mock(sha256=self.fake_sha256, **{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError}) + self.sha256Default=sha256Default + + all_throw = unittest.mock.Mock(**{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError, 'sha256.side_effect': ValueError}) + self.all_throw=all_throw + + no_algorithms = unittest.mock.Mock() + del no_algorithms.md5 + del no_algorithms.sha1 + del no_algorithms.sha256 + del no_algorithms.nonexist + self.no_algorithms=no_algorithms + + unsupported_algorithm = unittest.mock.Mock(unsupported=self.fake_sha256) + del unsupported_algorithm.md5 + del unsupported_algorithm.sha1 + del unsupported_algorithm.sha256 + del unsupported_algorithm.unsupported + self.unsupported_algorithm=unsupported_algorithm + ############################### + + ############################### + # system version mocks + VersionInfo = namedtuple('VersionInfo', 'major minor micro releaselevel serial') + v3_8 = VersionInfo(3, 8, 199, 'super-beta', 1337) + v3_9 = VersionInfo(3, 9, 0, 'alpha', 0) + v4_8 = VersionInfo(4, 8, 0, 'final', 0) + + self.sys_v3_8 = unittest.mock.Mock(version_info=v3_8) + self.sys_v3_9 = unittest.mock.Mock(version_info=v3_9) + self.sys_v4_8 = unittest.mock.Mock(version_info=v4_8) + ############################### + + def test_basic_failover_bad_hashlib_hash_init(self) -> None: + """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), + the hash init function returns None""" + assert _attempt_init_of_python_3_9_hash_object(None) is None + + def test_basic_failover_bad_hashlib_hash_get(self) -> None: + """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), + the hash get function returns None""" + assert _attempt_get_hash_function("nonexist", self.no_algorithms) is None + + def test_usedforsecurity_flag_behavior(self) -> None: + """Test usedforsecurity flag -> should be set to 'True' on older versions of python, and 'False' on Python >= 3.9""" + for version, expected in { + self.sys_v3_8: (True, 'md5'), + self.sys_v3_9: (False, 'md5'), + self.sys_v4_8: (False, 'md5'), + }.items(): + assert _attempt_init_of_python_3_9_hash_object(self.fake_md5, version) == expected + + def test_automatic_default_to_md5(self) -> None: + """Test automatic default to md5 even if sha1 available""" + for version, expected in { + self.sys_v3_8: (True, 'md5'), + self.sys_v3_9: (False, 'md5'), + self.sys_v4_8: (False, 'md5'), + }.items(): + _set_allowed_viable_default_hashes(self.md5Default, version) + set_hash_format(None, self.md5Default, version) + assert _get_hash_object(None, self.md5Default, version) == expected + + def test_automatic_default_to_sha256(self) -> None: + """Test automatic default to sha256 if other algorithms available but throw""" + for version, expected in { + self.sys_v3_8: (True, 'sha256'), + self.sys_v3_9: (False, 'sha256'), + self.sys_v4_8: (False, 'sha256'), + }.items(): + _set_allowed_viable_default_hashes(self.sha256Default, version) + set_hash_format(None, self.sha256Default, version) + assert _get_hash_object(None, self.sha256Default, version) == expected + + def test_automatic_default_to_sha1(self) -> None: + """Test automatic default to sha1 if md5 is missing from hashlib entirely""" + for version, expected in { + self.sys_v3_8: (True, 'sha1'), + self.sys_v3_9: (False, 'sha1'), + self.sys_v4_8: (False, 'sha1'), + }.items(): + _set_allowed_viable_default_hashes(self.sha1Default, version) + set_hash_format(None, self.sha1Default, version) + assert _get_hash_object(None, self.sha1Default, version) == expected + + def test_no_available_algorithms(self) -> None: + """expect exceptions on no available algorithms or when all algorithms throw""" + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.no_algorithms) + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.all_throw) + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.unsupported_algorithm) + + def test_bad_algorithm_set_attempt(self) -> None: + """expect exceptions on user setting an unsupported algorithm selections, either by host or by SCons""" + + # nonexistant hash algorithm, not supported by SCons + _set_allowed_viable_default_hashes(self.md5Available) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'blah blah blah', hashlib_used=self.no_algorithms) + + # md5 is default-allowed, but in this case throws when we attempt to use it + _set_allowed_viable_default_hashes(self.md5Available) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) + + # user attempts to use an algorithm that isn't supported by their current system but is supported by SCons + _set_allowed_viable_default_hashes(self.sha1Default) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) + + # user attempts to use an algorithm that is supported by their current system but isn't supported by SCons + _set_allowed_viable_default_hashes(self.sha1Default) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'unsupported', hashlib_used=self.unsupported_algorithm) + + def tearDown(self) -> None: + """Return SCons back to the normal global state for the hashing functions.""" + _set_allowed_viable_default_hashes(hashlib, sys) + set_hash_format(None) + + +if __name__ == "__main__": + unittest.main() diff --git a/SCons/Util/sctypesTests.py b/SCons/Util/sctypesTests.py new file mode 100644 index 0000000000..92db7828b8 --- /dev/null +++ b/SCons/Util/sctypesTests.py @@ -0,0 +1,241 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import annotations + +import os +import unittest +from collections import UserDict, UserList, UserString + +from SCons.Util.sctypes import ( + get_env_bool, + get_environment_var, + get_os_env_bool, + is_Dict, + is_List, + is_String, + is_Tuple, + to_bytes, + to_str, + to_String, +) + +# These sctypes classes have no unit tests. +# Null, NullSeq + +class OsEnviron: + """Used to temporarily mock os.environ""" + + def __init__(self, environ) -> None: + self._environ = environ + + def start(self) -> None: + self._stored = os.environ + os.environ = self._environ + + def stop(self) -> None: + os.environ = self._stored + del self._stored + + def __enter__(self): + self.start() + return os.environ + + def __exit__(self, *args) -> None: + self.stop() + + +class get_env_boolTestCase(unittest.TestCase): + def test_missing(self) -> None: + env = {} + var = get_env_bool(env, 'FOO') + assert var is False, "var should be False, not %s" % repr(var) + env = {'FOO': '1'} + var = get_env_bool(env, 'BAR') + assert var is False, "var should be False, not %s" % repr(var) + + def test_true(self) -> None: + for arg in ['TRUE', 'True', 'true', + 'YES', 'Yes', 'yes', + 'Y', 'y', + 'ON', 'On', 'on', + '1', '20', '-1']: + env = {'FOO': arg} + var = get_env_bool(env, 'FOO') + assert var is True, 'var should be True, not %s' % repr(var) + + def test_false(self) -> None: + for arg in ['FALSE', 'False', 'false', + 'NO', 'No', 'no', + 'N', 'n', + 'OFF', 'Off', 'off', + '0']: + env = {'FOO': arg} + var = get_env_bool(env, 'FOO', True) + assert var is False, 'var should be True, not %s' % repr(var) + + def test_default(self) -> None: + env = {'FOO': 'other'} + var = get_env_bool(env, 'FOO', True) + assert var is True, 'var should be True, not %s' % repr(var) + var = get_env_bool(env, 'FOO', False) + assert var is False, 'var should be False, not %s' % repr(var) + + +class get_os_env_boolTestCase(unittest.TestCase): + def test_missing(self) -> None: + with OsEnviron({}): + var = get_os_env_bool('FOO') + assert var is False, "var should be False, not %s" % repr(var) + with OsEnviron({'FOO': '1'}): + var = get_os_env_bool('BAR') + assert var is False, "var should be False, not %s" % repr(var) + + def test_true(self) -> None: + for arg in ['TRUE', 'True', 'true', + 'YES', 'Yes', 'yes', + 'Y', 'y', + 'ON', 'On', 'on', + '1', '20', '-1']: + with OsEnviron({'FOO': arg}): + var = get_os_env_bool('FOO') + assert var is True, 'var should be True, not %s' % repr(var) + + def test_false(self) -> None: + for arg in ['FALSE', 'False', 'false', + 'NO', 'No', 'no', + 'N', 'n', + 'OFF', 'Off', 'off', + '0']: + with OsEnviron({'FOO': arg}): + var = get_os_env_bool('FOO', True) + assert var is False, 'var should be True, not %s' % repr(var) + + def test_default(self) -> None: + with OsEnviron({'FOO': 'other'}): + var = get_os_env_bool('FOO', True) + assert var is True, 'var should be True, not %s' % repr(var) + var = get_os_env_bool('FOO', False) + assert var is False, 'var should be False, not %s' % repr(var) + + +class TestSctypes(unittest.TestCase): + def test_is_Dict(self) -> None: + assert is_Dict({}) + assert is_Dict(UserDict()) + try: + class mydict(dict): + pass + except TypeError: + pass + else: + assert is_Dict(mydict({})) + assert not is_Dict([]) + assert not is_Dict(()) + assert not is_Dict("") + + + def test_is_List(self) -> None: + assert is_List([]) + assert is_List(UserList()) + try: + class mylist(list): + pass + except TypeError: + pass + else: + assert is_List(mylist([])) + assert not is_List(()) + assert not is_List({}) + assert not is_List("") + + def test_is_String(self) -> None: + assert is_String("") + assert is_String(UserString('')) + try: + class mystr(str): + pass + except TypeError: + pass + else: + assert is_String(mystr('')) + assert not is_String({}) + assert not is_String([]) + assert not is_String(()) + + def test_is_Tuple(self) -> None: + assert is_Tuple(()) + try: + class mytuple(tuple): + pass + except TypeError: + pass + else: + assert is_Tuple(mytuple(())) + assert not is_Tuple([]) + assert not is_Tuple({}) + assert not is_Tuple("") + + def test_to_Bytes(self) -> None: + """ Test the to_Bytes method""" + self.assertEqual(to_bytes('Hello'), + bytearray('Hello', 'utf-8'), + "Check that to_bytes creates byte array when presented with non byte string.") + + def test_to_String(self) -> None: + """Test the to_String() method.""" + assert to_String(1) == "1", to_String(1) + assert to_String([1, 2, 3]) == str([1, 2, 3]), to_String([1, 2, 3]) + assert to_String("foo") == "foo", to_String("foo") + assert to_String(None) == 'None' + # test low level string converters too + assert to_str(None) == 'None' + assert to_bytes(None) == b'None' + + s1 = UserString('blah') + assert to_String(s1) == s1, s1 + assert to_String(s1) == 'blah', s1 + + class Derived(UserString): + pass + + s2 = Derived('foo') + assert to_String(s2) == s2, s2 + assert to_String(s2) == 'foo', s2 + + def test_get_env_var(self) -> None: + """Testing get_environment_var().""" + assert get_environment_var("$FOO") == "FOO", get_environment_var("$FOO") + assert get_environment_var("${BAR}") == "BAR", get_environment_var("${BAR}") + assert get_environment_var("$FOO_BAR1234") == "FOO_BAR1234", get_environment_var("$FOO_BAR1234") + assert get_environment_var("${BAR_FOO1234}") == "BAR_FOO1234", get_environment_var("${BAR_FOO1234}") + assert get_environment_var("${BAR}FOO") is None, get_environment_var("${BAR}FOO") + assert get_environment_var("$BAR ") is None, get_environment_var("$BAR ") + assert get_environment_var("FOO$BAR") is None, get_environment_var("FOO$BAR") + assert get_environment_var("$FOO[0]") is None, get_environment_var("$FOO[0]") + assert get_environment_var("${some('complex expression')}") is None, get_environment_var( + "${some('complex expression')}") + + +if __name__ == "__main__": + unittest.main()