feat: add Cloudflare Worker to serve message detail pages from R2

.github/workflows/build.yml

@@ -171,6 +171,7 @@ jobs:
         echo "Pushing statics..."
         cd static
         aws s3 sync . s3://static/mailarchive/$PKG_VERSION --only-show-errors --checksum-algorithm CRC32
+        aws s3 cp mlarchive/html/message-detail.html s3://ml-templates/message-detail.html --only-show-errors --checksum-algorithm CRC32
 
     - name: Augment dockerignore for docker image build
       env:

backend/mlarchive/archive/mail.py

@@ -1008,4 +1008,3 @@ def write_msg(self, subdir=None):
         # store regular public messages
         else:
             store_file('ml-messages', blob_path, io.BytesIO(self.bytes), content_type='message/rfc822')
-            store_file('ml-messages-json', blob_path, io.BytesIO(self.archive_message.as_json().encode('utf-8')), content_type='application/json')

backend/mlarchive/archive/management/commands/create_cf_worker_templates.py

@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+
+from django.core.management.base import BaseCommand, CommandError
+from mlarchive.archive.utils import create_cf_worker_templates
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Create templates for Cloudflare workers"
+
+    def handle(self, *args, **options):
+        try:
+            create_cf_worker_templates()
+        except Exception as e:
+            logger.error(f'create cloudflare worker templates failed: {e}')
+            raise CommandError(f'Command failed. {e}')

backend/mlarchive/archive/models.py

@@ -221,7 +221,31 @@ def as_json(self, fields=None, exclude=None):
         if 'updated' in data:
             data['updated'] = self.updated.isoformat()
 
-        data['content'] = self.get_body()
+        if self.previous_in_list():
+            data['previous_in_list'] = self.previous_in_list().get_absolute_url()
+        else:
+            data['previous_in_list'] = ''
+        if self.next_in_list():
+            data['next_in_list'] = self.next_in_list().get_absolute_url()
+        else:
+            data['next_in_list'] = ''
+
+        if self.previous_in_thread():
+            data['previous_in_thread'] = self.previous_in_thread().get_absolute_url()
+        else:
+            data['previous_in_thread'] = ''
+        if self.next_in_thread():
+            data['next_in_thread'] = self.next_in_thread().get_absolute_url()
+        else:
+            data['next_in_thread'] = ''
+
+        data['date_index_url'] = self.get_date_index_url()
+        data['thread_index_url'] = self.get_thread_index_url()
+        data['static_date_index_url'] = self.get_static_date_index_url()
+        data['static_thread_index_url'] = self.get_static_thread_index_url()
+
+        data['body'] = self.get_body_html()
+        data['thread_snippet'] = self.get_thread_snippet()
 
         return json.dumps(data)
 

backend/mlarchive/archive/signals.py

@@ -1,3 +1,4 @@
+import io
 import logging
 import os
 import requests
@@ -17,6 +18,7 @@
 
 from mlarchive.archive.models import Message, EmailList
 from mlarchive.archive.backends.elasticsearch import ESBackend, get_identifier
+from mlarchive.archive.storage_utils import store_file, remove_from_storage, move_object
 from mlarchive.archive.utils import _export_lists
 from mlarchive.celeryapp import app
 
@@ -27,6 +29,21 @@
 # Signal Handlers
 # --------------------------------------------------
 
+@receiver(post_save, sender=Message)
+def _save_message_json(sender, instance, created, **kwargs):
+    '''Save ml-messages-json blob for use in Cloudflare worker edge response'''
+    if not instance.email_list.private and created:
+        store_file(
+            kind='ml-messages-json',
+            name=instance.get_blob_name(),
+            file=io.BytesIO(instance.as_json().encode('utf-8')),
+            allow_overwrite=True,
+            content_type='application/json'
+        )
+        if instance.thread_order > 0:
+            update_message_json_thread(instance)
+
+
 @receiver([post_save, post_delete], sender=EmailList)
 def _clear_lists_cache(sender, instance, **kwargs):
     """If EmailList object is saved or deleted remove the list cache entries
@@ -40,6 +57,7 @@ def _message_remove(sender, instance, **kwargs):
     """When messages are removed, via the admin page, we need to move the message
     archive file to the "_removed" directory and purge the cache
     """
+    # move file on filesystem
     path = instance.get_file_path()
     if not os.path.exists(path):
         return
@@ -53,6 +71,17 @@ def _message_remove(sender, instance, **kwargs):
     else:
         shutil.move(path, target_dir)
 
+    # move blob
+    if instance.email_list.private:
+        source = 'ml-messages-private'
+    else:
+        source = 'ml-messages'
+    move_object(instance.get_blob_name(), source, 'ml-messages-removed')
+
+    # delete blob from ml-messages-json bucket
+    # Ok if it's not there, a private message wouldn't be
+    remove_from_storage(kind='ml-messages-json', name=instance.get_blob_name(), warn_if_missing=False)
+
     logger.info('message file moved: {} => {}'.format(path, target_dir))
 
     # if message is first of many in thread, should reset thread.first before
@@ -134,6 +163,19 @@ def _flush_noauth_cache(email_list):
     cache.delete_many(keys)
 
 
+def update_message_json_thread(message):
+    '''Write ml-messages-json for all other messages in thread
+    TODO: consider alternatives like client retrieving thread instead of computing
+    '''
+    for msg in message.thread.message_set.exclude(pk=message.pk):
+        store_file(
+            kind='ml-messages-json',
+            name=msg.get_blob_name(),
+            file=io.BytesIO(msg.as_json().encode('utf-8')),
+            allow_overwrite=True,
+            content_type='application/json'
+        )
+
 # --------------------------------------------------
 # Classes
 # --------------------------------------------------

backend/mlarchive/archive/storage_utils.py

@@ -181,3 +181,18 @@ def get_unique_blob_name(prefix, bucket):
     msg = 'Blobstore Error: get_unique_blob_name() failed.'
     logger.error(msg)
     raise RuntimeError(msg)
+
+
+def move_object(key: str, source_bucket: str, target_bucket: str) -> None:
+    if settings.ENABLE_BLOBSTORAGE:
+        try:
+            store = _get_storage(target_bucket)
+            content = retrieve_bytes(source_bucket, key)
+            store_bytes(target_bucket, key, content=content)
+            assert exists_in_storage(target_bucket, key)
+            assert store.size(key) == len(content)
+            remove_from_storage(source_bucket, key)
+        except Exception as err:
+            logger.error(f"Blobstore Error: Failed to move {key} from {source_bucket} to {target_bucket} {repr(err)}")
+            raise
+    return

backend/mlarchive/archive/utils.py

@@ -12,13 +12,17 @@
 import subprocess
 import sys
 from collections import defaultdict
+from pathlib import Path
 
 import mailmanclient
 from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
 from django.core.cache import cache
 from django.http import HttpResponse
 from django.core.exceptions import ValidationError
 from django.core.validators import validate_email
+from django.template.loader import render_to_string
+from django.test import RequestFactory
 
 from mlarchive.archive.models import (EmailList, Subscriber, Redirect, UserEmail, MailmanMember,
     User, Message)
@@ -857,3 +861,32 @@ def import_message_blob(bucket, name):
             listname=list_name,
             private=is_private)
         logger.info(f'Archive message status: {name} {status}')
+
+
+def create_cf_worker_templates():
+    """Create message template for Cloudflare worker. Here we are mainly mapping django template
+    varaibles to cloudflare worker mustache variables"""
+    path = Path(settings.CF_WORKER_TEMPLATE_DIR, 'message-detail.html')
+    path.parent.mkdir(parents=True, exist_ok=True)
+    context = {}
+    context['server_mode'] = 'production'
+    context['queryid'] = None  # query based navigation turned off in generic template
+    # context['static_mode_enabled']  # provided by context processor
+    # pass request to enable context processors
+    msg = {}
+    msg['subject'] = '{{ subject }}'
+    msg['get_date_index_url'] = '{{{ date_index_url }}}'
+    msg['get_thread_index_url'] = '{{{ thread_index_url }}}'
+    msg['get_static_date_index_url'] = '{{{ static_date_index_url }}}'
+    msg['get_static_thread_index_url'] = '{{{ static_thread_index_url }}}'
+    msg['get_thread_snippet'] = '{{{ thread_snippet }}}'
+    msg['get_body_html'] = '{{{ body }}}'
+    context['msg'] = msg
+    context['previous_in_list'] = {'get_absolute_url': '{{{ previous_in_list }}}'}
+    context['next_in_list'] = {'get_absolute_url': '{{{ next_in_list }}}'}
+    context['previous_in_thread'] = {'get_absolute_url': '{{{ previous_in_thread }}}'}
+    context['next_in_thread'] = {'get_absolute_url': '{{{ next_in_thread }}}'}
+    request = RequestFactory().get('/')
+    request.user = AnonymousUser()
+    html = render_to_string('archive/detail.html', context, request=request)
+    path.write_text(html, encoding='utf-8')

backend/mlarchive/settings/base.py

@@ -258,6 +258,10 @@
     'ml-messages-spam': {
         "BACKEND": "mlarchive.blobdb.storage.BlobdbStorage",
         "OPTIONS": {"bucket_name": 'ml-messages-spam'},
+    },
+    'ml-templates': {
+        "BACKEND": "mlarchive.blobdb.storage.BlobdbStorage",
+        "OPTIONS": {"bucket_name": 'ml-templates'},
     }
 }
 
@@ -271,6 +275,7 @@
     "ml-messages-filtered",
     "ml-messages-dupes",
     "ml-messages-spam",
+    "ml-templates",
 ]
 
 ENABLE_BLOBSTORAGE = True
@@ -527,6 +532,7 @@
 CLOUDFLARE_AUTH_KEY = env("CLOUDFLARE_AUTH_KEY")
 CLOUDFLARE_ZONE_ID = env("CLOUDFLARE_ZONE_ID")
 CACHE_CONTROL_MAX_AGE = 60 * 60 * 24 * 7     # one week
+CF_WORKER_TEMPLATE_DIR = os.path.join(BASE_DIR, 'static/mlarchive/html')
 
 # OIDC SETTINGS
 OIDC_RP_CLIENT_ID = env('OIDC_RP_CLIENT_ID')

backend/mlarchive/tests/archive/storage_utils.py

@@ -2,7 +2,8 @@
 
 from django.core.files.storage import storages
 
-from mlarchive.archive.storage_utils import get_unique_blob_name
+from mlarchive.archive.storage_utils import (get_unique_blob_name, store_str, move_object,
+    exists_in_storage)
 
 
 @pytest.mark.django_db(transaction=True)
@@ -13,3 +14,16 @@ def test_get_unique_blob_name(client):
     storage = storages[bucket]
     assert blob_name.startswith(prefix)
     assert not storage.exists(blob_name)
+
+
+@pytest.mark.django_db(transaction=True)
+def test_move_object(client):
+    source = 'ml-messages'
+    target = 'ml-messages-removed'
+    key = 'acme/PjjZawcPwvGsK6zLLOc4DOVwA4w'
+    store_str(source, key, content='This is a test')
+    assert exists_in_storage(source, key)
+    assert not exists_in_storage(target, key)
+    move_object(key, source, target)
+    assert not exists_in_storage(source, key)
+    assert exists_in_storage(target, key)

backend/mlarchive/tests/archive/utils.py

@@ -21,7 +21,8 @@
     create_mbox_file, _get_lists_as_xml, get_subscribers, Subscriber,
     get_mailman_lists, get_membership, get_subscriber_counts, get_fqdn,
     update_mbox_files, _export_lists, move_list, remove_selected, mark_not_spam,
-    is_duplicate_message, is_mailman_footer, import_message_blob)
+    is_duplicate_message, is_mailman_footer, import_message_blob,
+    create_cf_worker_templates)
 from mlarchive.archive.models import User, Message, Redirect, MailmanMember, UserEmail
 from mlarchive.archive.mail import make_hash
 from mlarchive.archive.forms import AdvancedSearchForm
@@ -602,3 +603,10 @@ def test_is_mailman_footer_detection():
     parts = [part for part in msg10.walk()]
     assert is_mailman_footer(parts[-1]) is False
     mbox.close()
+
+
+def test_create_cf_worker_templates():
+    """Test the creation of the Cloudflare worker message edge template"""
+    create_cf_worker_templates()
+    path = os.path.join(settings.CF_WORKER_TEMPLATE_DIR, 'message-detail.html')
+    assert os.path.exists(path)

build/app/collectstatics.sh

@@ -13,6 +13,9 @@ cp docker/configs/docker_env .env
 # Install Python dependencies
 pip --disable-pip-version-check --no-cache-dir install -r requirements.txt
 
+# Pre create Cloudflare worker templates
+backend/manage.py create_cf_worker_templates --settings=mlarchive.settings.settings_collectstatics
+
 # Collect statics
 backend/manage.py collectstatic --settings=mlarchive.settings.settings_collectstatics
 

workers/messages/.gitignore

@@ -0,0 +1,8 @@
+node_modules/
+.wrangler/
+.dev.vars
+*.log
+dist/
+.env
+.env.*
+wrangler.toml.backup

workers/messages/.npmrc

@@ -0,0 +1,2 @@
+# Use public registry
+registry=https://registry.npmjs.org/

workers/messages/README.md

@@ -0,0 +1,43 @@
+<div align="center">
+
+<img src="logo.png" alt="Cloudflare Workers" height="125" />
+
+#### Cloudflare Worker Proxy for serving message files
+
+</div>
+
+## Summary
+
+This Cloudflare Worker processes requests for archive message details. It does the following:
+
+- if the request is unauthenticated:
+  - if message object exists in blob storage (name from URL: dnsop/rY-OYgyL59afmpApNrW3UPo5wuM)
+    - get json blob from storage
+    - use json as context for template and return HTML
+- else fetch response from source and return
+
+## Routes
+- /arch/msg/*
+- /arch/ajax/msg/* (future)
+
+## Bindings
+- R2 ml-templates
+- R2 ml-messages-json
+
+## Development
+
+### Setup
+
+Make sure you have Node.js 20.x or later installed first. Then clone the repository locally and run `npm install`.
+
+### Dev Mode
+
+Use the command `npm run dev` to start the dev server.
+
+Use a command like this to upload file to local R2 
+npx wrangler r2 object put ml-messages-json/dnsop/PxDc-GHOEmUhxElwrT49dqcRyag --file=test-data.json --local
+npx wrangler r2 object put ml-templates/message-detail.html --file=sample-template.html --local
+
+### Deployment
+
+Use the command `npm run deploy` to deploy the project to Cloudflare Workers.

workers/messages/basic-template.html

@@ -0,0 +1,5 @@
+<h1>This is the message</h1>
+
+<h3>{{ subject }}</h3>
+
+<p>{{ content }}</p>