From 9b4a2dc46f31b3791443520d4f031908f24a551e Mon Sep 17 00:00:00 2001 From: Andrew Asseily Date: Tue, 7 Apr 2026 11:38:43 -0400 Subject: [PATCH 1/8] Set owner-only (0600) permissions on streaming and S3 Select output files --- awscli/customizations/s3events.py | 13 +++++++++---- awscli/customizations/streamingoutputarg.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/awscli/customizations/s3events.py b/awscli/customizations/s3events.py index 2a0c31d307a7..704e635813fe 100644 --- a/awscli/customizations/s3events.py +++ b/awscli/customizations/s3events.py @@ -12,6 +12,8 @@ # language governing permissions and limitations under the License. """Add S3 specific event streaming output arg.""" +import os + from awscli.arguments import CustomArgument STREAM_HELP_TEXT = 'Filename where the records will be saved' @@ -59,8 +61,7 @@ def replace_event_stream_docs(help_command, **kwargs): # This should never happen, but in the rare case that it does # we should be raising something with a helpful error message. raise DocSectionNotFoundError( - 'Could not find the "output" section for the command: %s' - % help_command + f'Could not find the "output" section for the command: {help_command}' ) doc.write('======\nOutput\n======\n') doc.write( @@ -98,7 +99,7 @@ class S3SelectStreamOutputArgument(CustomArgument): _DOCUMENT_AS_REQUIRED = True def __init__(self, stream_key, session, **kwargs): - super(S3SelectStreamOutputArgument, self).__init__(**kwargs) + super().__init__(**kwargs) # This is the key in the response body where we can find the # streamed contents. self._stream_key = stream_key @@ -120,7 +121,11 @@ def save_file(self, parsed, **kwargs): if self._stream_key not in parsed: return event_stream = parsed[self._stream_key] - with open(self._output_file, 'wb') as fp: + fd = os.open( + self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 + ) + os.fchmod(fd, 0o600) + with os.fdopen(fd, 'wb') as fp: for event in event_stream: if 'Records' in event: fp.write(event['Records']['Payload']) diff --git a/awscli/customizations/streamingoutputarg.py b/awscli/customizations/streamingoutputarg.py index c4decbb6844d..8b5c5cd3a90a 100644 --- a/awscli/customizations/streamingoutputarg.py +++ b/awscli/customizations/streamingoutputarg.py @@ -10,6 +10,8 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os + from botocore.model import Shape from awscli.arguments import BaseCLIArgument @@ -92,7 +94,7 @@ def add_to_params(self, parameters, value): service_id = self._operation_model.service_model.service_id.hyphenize() operation_name = self._operation_model.name self._session.register( - 'after-call.%s.%s' % (service_id, operation_name), self.save_file + f'after-call.{service_id}.{operation_name}', self.save_file ) def save_file(self, parsed, **kwargs): @@ -104,7 +106,11 @@ def save_file(self, parsed, **kwargs): return body = parsed[self._response_key] buffer_size = self._buffer_size - with open(self._output_file, 'wb') as fp: + fd = os.open( + self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 + ) + os.fchmod(fd, 0o600) + with os.fdopen(fd, 'wb') as fp: data = body.read(buffer_size) while data: fp.write(data) From 5c33c3021ae8d7f250a6277b12d93da11c3ed1df Mon Sep 17 00:00:00 2001 From: Andrew Asseily Date: Tue, 7 Apr 2026 11:40:38 -0400 Subject: [PATCH 2/8] Add tests to verify output file permissions are set to 0600 --- .../s3api/test_select_object_content.py | 24 +++++++++++++++++-- tests/functional/test_streaming_output.py | 19 +++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/tests/functional/s3api/test_select_object_content.py b/tests/functional/s3api/test_select_object_content.py index 04140563fa09..18025f745fdf 100644 --- a/tests/functional/s3api/test_select_object_content.py +++ b/tests/functional/s3api/test_select_object_content.py @@ -22,12 +22,12 @@ class TestGetObject(BaseAWSCommandParamsTest): prefix = ['s3api', 'select-object-content'] def setUp(self): - super(TestGetObject, self).setUp() + super().setUp() self.parsed_response = {'Payload': self.create_fake_payload()} self._tempdir = tempfile.mkdtemp() def tearDown(self): - super(TestGetObject, self).tearDown() + super().tearDown() shutil.rmtree(self._tempdir) def create_fake_payload(self): @@ -82,6 +82,26 @@ def test_can_stream_to_file(self): contents = f.read() self.assertEqual(contents, ('a,b,c,d\n' 'e,f,g,h\n')) + def test_output_file_permissions(self): + filename = os.path.join(self._tempdir, 'outfile_perms') + cmdline = self.prefix + [ + '--bucket', + 'mybucket', + '--key', + 'mykey', + '--expression', + 'SELECT * FROM S3Object', + '--expression-type', + 'SQL', + '--input-serialization', + '{"CSV": {}}', + '--output-serialization', + '{"CSV": {}}', + filename, + ] + self.assert_params_for_cmd(cmdline, ignore_params=True) + self.assertEqual(os.stat(filename).st_mode & 0o777, 0o600) + def test_errors_are_propagated(self): self.http_response.status_code = 400 self.parsed_response = { diff --git a/tests/functional/test_streaming_output.py b/tests/functional/test_streaming_output.py index 10aa6c74407e..e6cb8036ae63 100644 --- a/tests/functional/test_streaming_output.py +++ b/tests/functional/test_streaming_output.py @@ -11,17 +11,19 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os + from awscli.compat import BytesIO from awscli.testutils import BaseAWSCommandParamsTest, FileCreator class TestStreamingOutput(BaseAWSCommandParamsTest): def setUp(self): - super(TestStreamingOutput, self).setUp() + super().setUp() self.files = FileCreator() def tearDown(self): - super(TestStreamingOutput, self).tearDown() + super().tearDown() self.files.remove_all() def test_get_media_streaming_output(self): @@ -41,3 +43,16 @@ def test_get_media_streaming_output(self): self.assert_params_for_cmd(cmdline % outpath, params) with open(outpath, 'rb') as outfile: self.assertEqual(outfile.read(), b'testbody') + + def test_streaming_output_file_permissions(self): + cmdline = ( + 'kinesis-video-media get-media --stream-name test-stream ' + '--start-selector StartSelectorType=EARLIEST %s' + ) + self.parsed_response = { + 'ContentType': 'video/webm', + 'Payload': BytesIO(b'testbody'), + } + outpath = self.files.full_path('outfile') + self.assert_params_for_cmd(cmdline % outpath, ignore_params=True) + self.assertEqual(os.stat(outpath).st_mode & 0o777, 0o600) From bf83933b261149e66600eb11b4a9a9324afe0daa Mon Sep 17 00:00:00 2001 From: Andrew Asseily Date: Tue, 7 Apr 2026 11:44:17 -0400 Subject: [PATCH 3/8] Add changelog entry for file permissions --- .changes/next-release/bugfix-s3streamingoutput-59585.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/next-release/bugfix-s3streamingoutput-59585.json diff --git a/.changes/next-release/bugfix-s3streamingoutput-59585.json b/.changes/next-release/bugfix-s3streamingoutput-59585.json new file mode 100644 index 000000000000..e34eeb66de1b --- /dev/null +++ b/.changes/next-release/bugfix-s3streamingoutput-59585.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "s3, streaming output", + "description": "Output files created by S3 Select and streaming output commands are now created with owner-only permissions (0600). Existing files are also tightened to 0600 when overwritten." +} From 5d841e4ed1ba4e5c2db6495fa8953ce565050976 Mon Sep 17 00:00:00 2001 From: Andrew Asseily Date: Tue, 7 Apr 2026 13:46:22 -0400 Subject: [PATCH 4/8] Use os.chmod instead of os.fchmod --- awscli/customizations/s3events.py | 2 +- awscli/customizations/streamingoutputarg.py | 2 +- tests/functional/s3api/test_select_object_content.py | 7 ++++++- tests/functional/test_streaming_output.py | 7 ++++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/awscli/customizations/s3events.py b/awscli/customizations/s3events.py index 704e635813fe..2cd57fef7975 100644 --- a/awscli/customizations/s3events.py +++ b/awscli/customizations/s3events.py @@ -124,7 +124,7 @@ def save_file(self, parsed, **kwargs): fd = os.open( self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 ) - os.fchmod(fd, 0o600) + os.chmod(self._output_file, 0o600) with os.fdopen(fd, 'wb') as fp: for event in event_stream: if 'Records' in event: diff --git a/awscli/customizations/streamingoutputarg.py b/awscli/customizations/streamingoutputarg.py index 8b5c5cd3a90a..596aab8a3342 100644 --- a/awscli/customizations/streamingoutputarg.py +++ b/awscli/customizations/streamingoutputarg.py @@ -109,7 +109,7 @@ def save_file(self, parsed, **kwargs): fd = os.open( self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 ) - os.fchmod(fd, 0o600) + os.chmod(self._output_file, 0o600) with os.fdopen(fd, 'wb') as fp: data = body.read(buffer_size) while data: diff --git a/tests/functional/s3api/test_select_object_content.py b/tests/functional/s3api/test_select_object_content.py index 18025f745fdf..d643d2843e9e 100644 --- a/tests/functional/s3api/test_select_object_content.py +++ b/tests/functional/s3api/test_select_object_content.py @@ -15,7 +15,11 @@ import shutil import tempfile -from awscli.testutils import BaseAWSCommandParamsTest, BaseAWSHelpOutputTest +from awscli.testutils import ( + BaseAWSCommandParamsTest, + BaseAWSHelpOutputTest, + skip_if_windows, +) class TestGetObject(BaseAWSCommandParamsTest): @@ -82,6 +86,7 @@ def test_can_stream_to_file(self): contents = f.read() self.assertEqual(contents, ('a,b,c,d\n' 'e,f,g,h\n')) + @skip_if_windows('chmod is not supported on Windows') def test_output_file_permissions(self): filename = os.path.join(self._tempdir, 'outfile_perms') cmdline = self.prefix + [ diff --git a/tests/functional/test_streaming_output.py b/tests/functional/test_streaming_output.py index e6cb8036ae63..0a86c0bdc48b 100644 --- a/tests/functional/test_streaming_output.py +++ b/tests/functional/test_streaming_output.py @@ -14,7 +14,11 @@ import os from awscli.compat import BytesIO -from awscli.testutils import BaseAWSCommandParamsTest, FileCreator +from awscli.testutils import ( + BaseAWSCommandParamsTest, + FileCreator, + skip_if_windows, +) class TestStreamingOutput(BaseAWSCommandParamsTest): @@ -44,6 +48,7 @@ def test_get_media_streaming_output(self): with open(outpath, 'rb') as outfile: self.assertEqual(outfile.read(), b'testbody') + @skip_if_windows('chmod is not supported on Windows') def test_streaming_output_file_permissions(self): cmdline = ( 'kinesis-video-media get-media --stream-name test-stream ' From 538b1de6f20444a8c3499c56d886e90c483ed23f Mon Sep 17 00:00:00 2001 From: Andrew Asseily Date: Thu, 9 Apr 2026 13:33:27 -0400 Subject: [PATCH 5/8] Add comment explaining permission bitmask in tests --- tests/functional/s3api/test_select_object_content.py | 1 + tests/functional/test_streaming_output.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/functional/s3api/test_select_object_content.py b/tests/functional/s3api/test_select_object_content.py index d643d2843e9e..e9078aa4b677 100644 --- a/tests/functional/s3api/test_select_object_content.py +++ b/tests/functional/s3api/test_select_object_content.py @@ -105,6 +105,7 @@ def test_output_file_permissions(self): filename, ] self.assert_params_for_cmd(cmdline, ignore_params=True) + # Mask file type bits to isolate permission bits (rwxrwxrwx) self.assertEqual(os.stat(filename).st_mode & 0o777, 0o600) def test_errors_are_propagated(self): diff --git a/tests/functional/test_streaming_output.py b/tests/functional/test_streaming_output.py index 0a86c0bdc48b..3255e7b74e9a 100644 --- a/tests/functional/test_streaming_output.py +++ b/tests/functional/test_streaming_output.py @@ -60,4 +60,5 @@ def test_streaming_output_file_permissions(self): } outpath = self.files.full_path('outfile') self.assert_params_for_cmd(cmdline % outpath, ignore_params=True) + # Mask file type bits to isolate permission bits (rwxrwxrwx) self.assertEqual(os.stat(outpath).st_mode & 0o777, 0o600) From 2a91f0f3f18fafe2727270afdc4e03abcbe184c7 Mon Sep 17 00:00:00 2001 From: Andrew Asseily Date: Fri, 10 Apr 2026 11:43:33 -0400 Subject: [PATCH 6/8] Skip chmod on non-regular files like /dev/null in streaming output --- awscli/customizations/s3events.py | 5 ++++- awscli/customizations/streamingoutputarg.py | 5 ++++- .../s3api/test_select_object_content.py | 21 +++++++++++++++++++ tests/functional/test_streaming_output.py | 14 +++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/awscli/customizations/s3events.py b/awscli/customizations/s3events.py index 2cd57fef7975..5e0b66d35823 100644 --- a/awscli/customizations/s3events.py +++ b/awscli/customizations/s3events.py @@ -13,6 +13,7 @@ """Add S3 specific event streaming output arg.""" import os +import stat from awscli.arguments import CustomArgument @@ -124,7 +125,9 @@ def save_file(self, parsed, **kwargs): fd = os.open( self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 ) - os.chmod(self._output_file, 0o600) + if stat.S_ISREG(os.fstat(fd).st_mode): + # Only chmod regular files; skip devices like /dev/null + os.chmod(self._output_file, 0o600) with os.fdopen(fd, 'wb') as fp: for event in event_stream: if 'Records' in event: diff --git a/awscli/customizations/streamingoutputarg.py b/awscli/customizations/streamingoutputarg.py index 596aab8a3342..581785c65372 100644 --- a/awscli/customizations/streamingoutputarg.py +++ b/awscli/customizations/streamingoutputarg.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import os +import stat from botocore.model import Shape @@ -109,7 +110,9 @@ def save_file(self, parsed, **kwargs): fd = os.open( self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 ) - os.chmod(self._output_file, 0o600) + if stat.S_ISREG(os.fstat(fd).st_mode): + # Only chmod regular files; skip devices like /dev/null + os.chmod(self._output_file, 0o600) with os.fdopen(fd, 'wb') as fp: data = body.read(buffer_size) while data: diff --git a/tests/functional/s3api/test_select_object_content.py b/tests/functional/s3api/test_select_object_content.py index e9078aa4b677..b5b7c33b48a0 100644 --- a/tests/functional/s3api/test_select_object_content.py +++ b/tests/functional/s3api/test_select_object_content.py @@ -108,6 +108,27 @@ def test_output_file_permissions(self): # Mask file type bits to isolate permission bits (rwxrwxrwx) self.assertEqual(os.stat(filename).st_mode & 0o777, 0o600) + @skip_if_windows('chmod is not supported on Windows') + def test_output_does_not_chmod_non_regular_files(self): + cmdline = self.prefix + [ + '--bucket', + 'mybucket', + '--key', + 'mykey', + '--expression', + 'SELECT * FROM S3Object', + '--expression-type', + 'SQL', + '--input-serialization', + '{"CSV": {}}', + '--output-serialization', + '{"CSV": {}}', + '/dev/null', + ] + original_mode = os.stat('/dev/null').st_mode & 0o777 + self.assert_params_for_cmd(cmdline, ignore_params=True) + self.assertEqual(os.stat('/dev/null').st_mode & 0o777, original_mode) + def test_errors_are_propagated(self): self.http_response.status_code = 400 self.parsed_response = { diff --git a/tests/functional/test_streaming_output.py b/tests/functional/test_streaming_output.py index 3255e7b74e9a..7fbb9980a6a0 100644 --- a/tests/functional/test_streaming_output.py +++ b/tests/functional/test_streaming_output.py @@ -62,3 +62,17 @@ def test_streaming_output_file_permissions(self): self.assert_params_for_cmd(cmdline % outpath, ignore_params=True) # Mask file type bits to isolate permission bits (rwxrwxrwx) self.assertEqual(os.stat(outpath).st_mode & 0o777, 0o600) + + @skip_if_windows('chmod is not supported on Windows') + def test_streaming_output_does_not_chmod_non_regular_files(self): + cmdline = ( + 'kinesis-video-media get-media --stream-name test-stream ' + '--start-selector StartSelectorType=EARLIEST %s' + ) + self.parsed_response = { + 'ContentType': 'video/webm', + 'Payload': BytesIO(b'testbody'), + } + original_mode = os.stat('/dev/null').st_mode & 0o777 + self.assert_params_for_cmd(cmdline % '/dev/null', ignore_params=True) + self.assertEqual(os.stat('/dev/null').st_mode & 0o777, original_mode) From 399fd5290c44c553471d7dea875f59c4b78fd393 Mon Sep 17 00:00:00 2001 From: Andrew Asseily <77591070+AndrewAsseily@users.noreply.github.com> Date: Fri, 10 Apr 2026 11:55:23 -0400 Subject: [PATCH 7/8] Fix comment formatting in s3events.py --- awscli/customizations/s3events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscli/customizations/s3events.py b/awscli/customizations/s3events.py index 5e0b66d35823..50818865e31b 100644 --- a/awscli/customizations/s3events.py +++ b/awscli/customizations/s3events.py @@ -126,7 +126,7 @@ def save_file(self, parsed, **kwargs): self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 ) if stat.S_ISREG(os.fstat(fd).st_mode): - # Only chmod regular files; skip devices like /dev/null + # Only chmod regular files; skip special files like /dev/null os.chmod(self._output_file, 0o600) with os.fdopen(fd, 'wb') as fp: for event in event_stream: From b657820c616e98ebaf58479800e41a5a48755532 Mon Sep 17 00:00:00 2001 From: Andrew Asseily <77591070+AndrewAsseily@users.noreply.github.com> Date: Fri, 10 Apr 2026 11:55:55 -0400 Subject: [PATCH 8/8] Update comment for chmod operation Clarified comment to specify skipping special files. --- awscli/customizations/streamingoutputarg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awscli/customizations/streamingoutputarg.py b/awscli/customizations/streamingoutputarg.py index 581785c65372..fe23bd7a5195 100644 --- a/awscli/customizations/streamingoutputarg.py +++ b/awscli/customizations/streamingoutputarg.py @@ -111,7 +111,7 @@ def save_file(self, parsed, **kwargs): self._output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 ) if stat.S_ISREG(os.fstat(fd).st_mode): - # Only chmod regular files; skip devices like /dev/null + # Only chmod regular files; skip special files like /dev/null os.chmod(self._output_file, 0o600) with os.fdopen(fd, 'wb') as fp: data = body.read(buffer_size)