From 966eaf0bccb07d92dcaaf3e9b161f4bf99faa0ee Mon Sep 17 00:00:00 2001
From: "David G. Johnston" <david.g.johnston@gmail.com>
Date: Mon, 28 Apr 2025 17:22:51 -0700
Subject: [PATCH 1/2] Create new patches from pgarchives

Built without the proposed Workflow changes since this patch set
is intended to demonstrate the integration of CFBot into CFApp.

This first commit simply sets up the endpoint for pgarchives to
send a complete Thread with Patchset to CFApp and have a new patch
created.  This allows for real demos without having to setup dummy
data - though the later will be part of it as well.

The Patch screen thread display area has been reworked a bit, in
particular to accomodate the added contextual data available such
as is_patch and the specific thread message containing patches.
The model now also handles multiple attachments per thread message
by removing the too-constraining unique together specification.
The patch itself tracks which message contains its most recent
patch set and returns only those attachments upon request.
---
 Makefile                                      |  7 ++
 pgcommitfest/commitfest/apiv1.py              | 78 +++++++++++++++++++
 .../commitfest/fixtures/minimal_data.json     | 19 +++++
 .../0011_accept_multiple_attachments.py       | 47 +++++++++++
 pgcommitfest/commitfest/models.py             | 33 +++++++-
 pgcommitfest/commitfest/templates/patch.html  | 16 ++--
 pgcommitfest/commitfest/util.py               | 19 +++++
 pgcommitfest/commitfest/views.py              |  1 +
 pgcommitfest/settings.py                      |  2 +
 pgcommitfest/urls.py                          |  4 +-
 10 files changed, 218 insertions(+), 8 deletions(-)
 create mode 100644 pgcommitfest/commitfest/apiv1.py
 create mode 100644 pgcommitfest/commitfest/fixtures/minimal_data.json
 create mode 100644 pgcommitfest/commitfest/migrations/0011_accept_multiple_attachments.py

diff --git a/Makefile b/Makefile
index c034382..a986a9c 100644
--- a/Makefile
+++ b/Makefile
@@ -16,3 +16,10 @@ lint-fix-unsafe:
 	npx @biomejs/biome check --fix --unsafe
 
 fix: format lint-fix-unsafe
+
+init-dev:
+	dropdb --if-exists pgcommitfest
+	createdb pgcommitfest
+	./manage.py migrate
+	./manage.py loaddata auth_data.json
+	./manage.py loaddata minimal_data.json
diff --git a/pgcommitfest/commitfest/apiv1.py b/pgcommitfest/commitfest/apiv1.py
new file mode 100644
index 0000000..d045cb2
--- /dev/null
+++ b/pgcommitfest/commitfest/apiv1.py
@@ -0,0 +1,78 @@
+from django.db import transaction
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_POST
+
+import json
+from datetime import datetime
+
+from .models import (
+    CommitFest,
+    MailThread,
+    MailThreadAttachment,
+    Patch,
+    PatchHistory,
+    PatchOnCommitFest,
+    Topic,
+)
+from .util import apiResponse
+
+
+@csrf_exempt
+@require_POST
+@transaction.atomic
+def create_patch(request):
+    """
+    Create a new patch with placeholder values for required fields.
+    """
+    body_string = request.body.decode("utf-8")
+    body_json = json.loads(body_string)
+
+    target_cf = CommitFest.objects.filter(status=CommitFest.STATUS_OPEN).first()
+
+    mailthread = MailThread.objects.create(
+        messageid=body_json.get("thread_message_id"),
+        subject=body_json.get("thread_subject_line"),
+        firstmessage=body_json.get("thread_message_date"),
+        firstauthor=body_json.get("thread_from_author"),
+        latestmsgid=body_json.get("most_recent_message_id"),
+        latestmessage=body_json.get("most_recent_message_date"),
+        latestauthor=body_json.get("most_recent_from_author"),
+        latestsubject=body_json.get("most_recent_subject_line"),
+        patchsetmsgid=body_json.get("patch_message_id"),
+    )
+
+    for attachment in body_json.get("fileset"):
+        MailThreadAttachment.objects.get_or_create(
+            mailthread=mailthread,
+            messageid=body_json.get("patch_message_id"),
+            attachmentid=attachment["attachment_id"],
+            filename=attachment["filename"],
+            contenttype=attachment["content_type"],
+            ispatch=attachment["is_patch"],
+            author=body_json.get("patch_from_author"),
+            date=body_json.get("patch_message_date"),
+        )
+
+    topic = Topic.objects.filter(id=1).first()  # Miscellaneous for now
+    # Create a new Patch instance with required fields set to None
+    patch = Patch.objects.create(
+        name=body_json.get("thread_subject_line"),
+        topic=topic,
+        patchset_messageid=body_json.get("patch_message_id", None),
+        patchset_messagedate=body_json.get("patch_message_date", None),
+        lastmail=body_json.get("most_recent_message_date", None),
+    )
+
+    poc = PatchOnCommitFest(patch=patch, commitfest=target_cf, enterdate=datetime.now())
+    poc.save()
+    PatchHistory(
+        patch=patch, by_cfbot=True, what="Patch created from mail thread"
+    ).save()
+
+    mailthread.patches.add(patch)
+    mailthread.save()
+
+    return apiResponse(
+        request,
+        {"patch_id": patch.id, "message": "Patch created with placeholder values."},
+    )
diff --git a/pgcommitfest/commitfest/fixtures/minimal_data.json b/pgcommitfest/commitfest/fixtures/minimal_data.json
new file mode 100644
index 0000000..5c311a7
--- /dev/null
+++ b/pgcommitfest/commitfest/fixtures/minimal_data.json
@@ -0,0 +1,19 @@
+[
+    {
+        "model": "commitfest.topic",
+        "pk": 1,
+        "fields": {
+            "topic": "Not Classified"
+        }
+    },
+    {
+        "model": "commitfest.commitfest",
+        "pk": 1,
+        "fields": {
+            "name": "2025-04",
+            "status": 2,
+            "startdate": "2025-04-01",
+            "enddate": "2025-04-30"
+        }
+}
+]
diff --git a/pgcommitfest/commitfest/migrations/0011_accept_multiple_attachments.py b/pgcommitfest/commitfest/migrations/0011_accept_multiple_attachments.py
new file mode 100644
index 0000000..8bc1d5b
--- /dev/null
+++ b/pgcommitfest/commitfest/migrations/0011_accept_multiple_attachments.py
@@ -0,0 +1,47 @@
+# Generated by Django 4.2.20 on 2025-04-29 00:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("commitfest", "0010_add_failing_since_column"),
+    ]
+    operations = [
+        migrations.AlterModelOptions(
+            name="mailthreadattachment",
+            options={
+                "ordering": (
+                    "-date",
+                    "messageid",
+                    "ispatch",
+                    "filename",
+                    "attachmentid",
+                )
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name="mailthreadattachment",
+            unique_together={("mailthread", "messageid", "attachmentid")},
+        ),
+        migrations.AddField(
+            model_name="mailthreadattachment",
+            name="contenttype",
+            field=models.CharField(max_length=1000, null=True),
+        ),
+        migrations.AddField(
+            model_name="patch",
+            name="patchset_messagedate",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="patch",
+            name="patchset_messageid",
+            field=models.CharField(max_length=1000, null=True),
+        ),
+        migrations.AddField(
+            model_name="mailthread",
+            name="patchsetmsgid",
+            field=models.CharField(max_length=1000, null=True),
+        ),
+    ]
diff --git a/pgcommitfest/commitfest/models.py b/pgcommitfest/commitfest/models.py
index fcd9edb..cd2eb2e 100644
--- a/pgcommitfest/commitfest/models.py
+++ b/pgcommitfest/commitfest/models.py
@@ -153,6 +153,14 @@ class Patch(models.Model, DiffableModel):
     # that's attached to this message.
     lastmail = models.DateTimeField(blank=True, null=True)
 
+    # Pointer to the threadid/messageid containing the most recent patchset
+    patchset_messageid = models.CharField(
+        max_length=1000, blank=False, null=True, db_index=False
+    )
+    # Cache the timestamp of the patchset message to easily determine
+    # whether a new patchset message should replace it.
+    patchset_messagedate = models.DateTimeField(blank=True, null=True)
+
     map_manytomany_for_diff = {
         "authors": "authors_string",
         "reviewers": "reviewers_string",
@@ -208,6 +216,26 @@ class Patch(models.Model, DiffableModel):
         else:
             self.lastmail = max(threads, key=lambda t: t.latestmessage).latestmessage
 
+    # XXX: make messageid an optional input to return a non-current patchset or to
+    # facilitate confirmation that the current patch is the one being worked on...
+    def get_attachments(self):
+        """
+        Return the actual attachments for the patch.
+        """
+        return [
+            {
+                "attachmentid": attachment.attachmentid,
+                "filename": attachment.filename,
+                "contenttype": attachment.contenttype,
+                "ispatch": attachment.ispatch,
+                "author": attachment.author,
+                "date": attachment.date,
+            }
+            for attachment in MailThreadAttachment.objects.filter(
+                mailthread__patches=self, messageid=self.patchset_messageid
+            )
+        ]
+
     def __str__(self):
         return self.name
 
@@ -391,6 +419,7 @@ class MailThread(models.Model):
     latestauthor = models.CharField(max_length=500, null=False, blank=False)
     latestsubject = models.CharField(max_length=500, null=False, blank=False)
     latestmsgid = models.CharField(max_length=1000, null=False, blank=False)
+    patchsetmsgid = models.CharField(max_length=1000, null=True, blank=False)
 
     def __str__(self):
         return self.subject
@@ -409,13 +438,15 @@ class MailThreadAttachment(models.Model):
     date = models.DateTimeField(null=False, blank=False)
     author = models.CharField(max_length=500, null=False, blank=False)
     ispatch = models.BooleanField(null=True)
+    contenttype = models.CharField(max_length=1000, null=True, blank=False)
 
     class Meta:
-        ordering = ("-date",)
+        ordering = ("-date", "messageid", "ispatch", "filename", "attachmentid")
         unique_together = (
             (
                 "mailthread",
                 "messageid",
+                "attachmentid",
             ),
         )
 
diff --git a/pgcommitfest/commitfest/templates/patch.html b/pgcommitfest/commitfest/templates/patch.html
index 2aafd14..7104a39 100644
--- a/pgcommitfest/commitfest/templates/patch.html
+++ b/pgcommitfest/commitfest/templates/patch.html
@@ -79,6 +79,10 @@
     <th style="white-space: nowrap;">Latest email</th>
     <td>{%if patch.lastmail%}{{patch.lastmail}} ({% cfwhen patch.lastmail %}){%endif%}</td>
    </tr>
+   <tr>
+    <th style="white-space: nowrap;">Latest patch set</th>
+    <td>{%if patch.patchset_messageid%}{{patch.patchset_messagedate}} ({% cfwhen patch.patchset_messagedate %}) on {{patch.patchset_messageid}}{%endif%}</td>
+   </tr>
    <tr>
     <th>Status</th>
     <td>{%for c in patch_commitfests %}
@@ -121,16 +125,18 @@
     {%endif%}
     <dl>
      {%for t in patch.mailthread_set.all%}
-      <dt><a href="https://www.postgresql.org/message-id/flat/{{t.messageid}}">{{t.subject}}</a> <button type="button" class="close close-nofloat" title="Detach this thread" onclick="detachThread({{cf.id}},{{patch.id}},'{{t.messageid}}')">&times;</button></dt>
+      <dt><a href="{{settings.ARCHIVES_BASE}}/message-id/flat/{{t.messageid}}">{{t.subject}}</a> <button type="button" class="close close-nofloat" title="Detach this thread" onclick="detachThread({{cf.id}},{{patch.id}},'{{t.messageid}}')">&times;</button></dt>
       <dd>
-       First at <a href="https://www.postgresql.org/message-id/{{t.messageid}}">{{t.firstmessage}}</a> by {{t.firstauthor|hidemail}}<br/>
-       Latest at <a href="https://www.postgresql.org/message-id/{{t.latestmsgid}}">{{t.latestmessage}}</a> by {{t.latestauthor|hidemail}}<br/>
+       First message at <a href="{{settings.ARCHIVES_BASE}}/message-id/{{t.messageid}}">{{t.firstmessage}}</a> by {{t.firstauthor|hidemail}}<br/>
+       Latest message at <a href="{{settings.ARCHIVES_BASE}}/message-id/{{t.latestmsgid}}">{{t.latestmessage}}</a> by {{t.latestauthor|hidemail}}<br/>
        {%for ta in t.mailthreadattachment_set.all%}
         {%if forloop.first%}
-         Latest attachment (<a href="https://www.postgresql.org/message-id/attachment/{{ta.attachmentid}}/{{ta.filename}}">{{ta.filename}}</a>) at <a href="https://www.postgresql.org/message-id/{{ta.messageid}}">{{ta.date}}</a> from {{ta.author|hidemail}} <button type="button" class="btn btn-default btn-xs" data-toggle="collapse" data-target="#att{{t.pk}}" title="Show all attachments"><i class="glyphicon glyphicon-plus"></i></button>
+         Latest files at <a href="{{settings.ARCHIVES_BASE}}/message-id/{{ta.messageid}}">{{ta.date}}</a> from {{ta.author|hidemail}} <button type="button" class="btn btn-default btn-xs" data-toggle="collapse" data-target="#att{{t.pk}}" title="Show all attachments"><i class="glyphicon glyphicon-plus"></i></button>
          <div id="att{{t.pk}}" class="collapse">
         {%endif%}
-        &nbsp;&nbsp;&nbsp;&nbsp;Attachment (<a href="https://www.postgresql.org/message-id/attachment/{{ta.attachmentid}}/{{ta.filename}}">{{ta.filename}}</a>) at <a href="https://www.postgresql.org/message-id/{{ta.messageid}}">{{ta.date}}</a> from {{ta.author|hidemail}} (Patch: {{ta.ispatch|yesno:"Yes,No,Pending check"}})<br/>
+        {%if True %}
+         &nbsp;&nbsp;&nbsp;&nbsp;{{ta.ispatch|yesno:"Patch,File,Unknown"}} <a href="{{settings.ARCHIVES_BASE}}/message-id/attachment/{{ta.attachmentid}}/{{ta.filename}}">{{ta.filename}}</a> ({{ta.contenttype}})<br/>
+        {%endif%}
         {%if forloop.last%}</div>{%endif%}
        {%endfor%}
        <div>
diff --git a/pgcommitfest/commitfest/util.py b/pgcommitfest/commitfest/util.py
index 10f9c81..e449a01 100644
--- a/pgcommitfest/commitfest/util.py
+++ b/pgcommitfest/commitfest/util.py
@@ -1,5 +1,9 @@
 import django.db.models.fields.related
 from django.forms.models import model_to_dict
+from django.http import HttpResponse
+
+import json
+from datetime import datetime
 
 
 class DiffableModel(object):
@@ -44,3 +48,18 @@ class DiffableModel(object):
         fields = [field.name for field in self._meta.fields]
         fields.extend([field.name for field in self._meta.many_to_many])
         return model_to_dict(self, fields=fields)
+
+
+def datetime_serializer(obj):
+    if isinstance(obj, datetime):
+        return obj.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
+    raise TypeError("Type not serializable")
+
+
+def apiResponse(request, payload, status=200, content_type="application/json"):
+    response = HttpResponse(
+        json.dumps(payload, default=datetime_serializer), status=status
+    )
+    response["Content-Type"] = content_type
+    response["Access-Control-Allow-Origin"] = "*"
+    return response
diff --git a/pgcommitfest/commitfest/views.py b/pgcommitfest/commitfest/views.py
index 7b6dd63..fcbb86d 100644
--- a/pgcommitfest/commitfest/views.py
+++ b/pgcommitfest/commitfest/views.py
@@ -708,6 +708,7 @@ def patch(request, patchid):
                 {"title": cf.title, "href": "/%s/" % cf.pk},
             ],
             "userprofile": getattr(request.user, "userprofile", UserProfile()),
+            "settings": settings,
         },
     )
 
diff --git a/pgcommitfest/settings.py b/pgcommitfest/settings.py
index 9b867b7..c1b8e5c 100644
--- a/pgcommitfest/settings.py
+++ b/pgcommitfest/settings.py
@@ -155,6 +155,8 @@ LOGIN_URL = "/account/login/"
 ARCHIVES_TIMEOUT = 10  # Seconds to wait for calls to the archives
 ARCHIVES_SERVER = "localhost"
 ARCHIVES_PORT = "8001"
+ARCHIVES_PROTO = "https" if ARCHIVES_PORT == "443" else "http"
+ARCHIVES_BASE = ARCHIVES_PROTO + "://" + ARCHIVES_SERVER + ":" + ARCHIVES_PORT
 ARCHIVES_HOST = "archives.postgresql.org"  # Host: header to send
 ARCHIVES_APIKEY = None
 
diff --git a/pgcommitfest/urls.py b/pgcommitfest/urls.py
index a67f55f..d4efbd2 100644
--- a/pgcommitfest/urls.py
+++ b/pgcommitfest/urls.py
@@ -3,18 +3,18 @@ from django.urls import re_path
 
 import pgcommitfest.auth
 import pgcommitfest.commitfest.ajax as ajax
+import pgcommitfest.commitfest.apiv1 as apiv1
 import pgcommitfest.commitfest.lookups as lookups
 import pgcommitfest.commitfest.reports as reports
 import pgcommitfest.commitfest.views as views
 import pgcommitfest.userprofile.views
 
-# Uncomment the next two lines to enable the admin:
-# from django.contrib import admin
 admin.autodiscover()
 
 
 urlpatterns = [
     re_path(r"^$", views.home),
+    re_path(r"^api/test/cfapp/create_patch$", apiv1.create_patch),
     re_path(r"^me/$", views.me),
     re_path(r"^archive/$", views.archive),
     re_path(r"^activity(?P<rss>\.rss)?/", views.activity),
-- 
2.34.1

