merge upstream

Author: rudyfraser

.github/workflows/cypress-tests.yml

@@ -34,7 +34,7 @@ jobs:
         mkcert -cert-file localhost.pem -key-file localhost-key.pem \
           localhost 127.0.0.1 ::1 oidc-simulator host.docker.internal
         
-        # Copy mkcert root CA to the certs directory for Docker containers
+        # Copy mkcert root CA for other containers
         cp "$(mkcert -CAROOT)/rootCA.pem" ./rootCA.pem
         
         # List generated files for debugging
@@ -89,11 +89,14 @@ jobs:
       if: always()
       run: |
         # Show logs from critical services for debugging
+        echo "=== NGINX Proxy logs ==="
+        docker compose -f docker-compose.test.yml logs nginx-proxy || true
+
         echo "=== OIDC Simulator logs ==="
-        docker compose -f docker-compose.test.yml logs oidc-simulator | tail -100
+        docker compose -f docker-compose.test.yml logs oidc-simulator || true
         
         echo "=== Server logs ==="
-        docker compose -f docker-compose.test.yml logs server | tail -100
+        docker compose -f docker-compose.test.yml logs server || true
 
     - name: Run auth setup test
       run: |

.github/workflows/jest-server-test.yml

@@ -65,13 +65,13 @@ jobs:
       run: |
         # Build all services except server (tests run on host)
         docker compose -f docker-compose.test.yml --env-file test.env build \
-          math postgres file-server maildev oidc-simulator
+          math postgres file-server maildev oidc-simulator dynamodb
 
     - name: Start services
       run: |
         # Start only required services in detached mode (exclude server)
         docker compose -f docker-compose.test.yml --env-file test.env up -d \
-          math postgres file-server maildev oidc-simulator
+          math postgres file-server maildev oidc-simulator dynamodb
         
         # Wait for services to be ready
         echo "Waiting for services to start..."
@@ -88,6 +88,11 @@ jobs:
         
         echo "Checking oidc-simulator..."
         curl -k -f https://localhost:3000/.well-known/jwks.json || true
+        
+        echo "Checking DynamoDB..."
+        curl -f http://localhost:8000 || true
+        # Alternative check using AWS CLI if available
+        aws dynamodb list-tables --endpoint-url http://localhost:8000 --region us-east-1 2>/dev/null || true
 
     - name: Setup Node.js
       uses: actions/setup-node@v4
@@ -114,6 +119,7 @@ jobs:
         export DATABASE_URL=postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test
         export MAILDEV_HOST=localhost
         export STATIC_FILES_HOST=localhost
+        export DYNAMODB_ENDPOINT=http://localhost:8000
         
         # Run the tests on the host machine
         npm test -- --ci --coverage --maxWorkers=2
@@ -138,6 +144,9 @@ jobs:
         
         echo "=== Math logs ==="
         docker compose -f docker-compose.test.yml logs math | tail -50
+        
+        echo "=== DynamoDB logs ==="
+        docker compose -f docker-compose.test.yml logs dynamodb | tail -100
 
     - name: Clean up
       if: always()

Makefile

@@ -102,8 +102,8 @@ start-FULL-REBUILD: echo_vars stop rm-ALL ## Remove and restart all Docker conta
 	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} build --no-cache
 	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up ${DETACH_ARG}
 
-rebuild-web: echo_vars ## Rebuild and restart just the file-server container and its static assets
-	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up ${DETACH_ARG} --build --force-recreate file-server
+rebuild-web: echo_vars ## Rebuild and restart just the file-server container and its static assets, and client-participation-alpha
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up ${DETACH_ARG} --build --force-recreate file-server client-participation-alpha
 
 build-web-assets: ## Build and extract static web assets for cloud deployment to `build` dir
 	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} create --build --force-recreate file-server

cdk/launchTemplates.ts

@@ -30,21 +30,21 @@ export default (
   ollamaKeyPair: ec2.IKeyPair | undefined,
   ollamaSecurityGroup: ec2.ISecurityGroup
 ) => {
-  // Generic User Data function (Works with NAT Gateway for internet)
   const usrdata = (CLOUDWATCH_LOG_GROUP_NAME: string, service: string, instanceSize?: string) => {
     let ld: ec2.UserData;
     ld = ec2.UserData.forLinux();
+    const persistentConfigDir = '/etc/app-info';
     ld.addCommands(
       '#!/bin/bash',
       'set -e',
       'set -x',
-      `echo "Writing service type '${service}' to /tmp/service_type.txt"`,
-      `echo "${service}" > /tmp/service_type.txt`,
-      `echo "Contents of /tmp/service_type.txt: $(cat /tmp/service_type.txt)"`,
-      // If instanceSize is provided, write it to a file
-      instanceSize ? `echo "Writing instance size '${instanceSize}' to /tmp/instance_size.txt"` : '',
-      instanceSize ? `echo "${instanceSize}" > /tmp/instance_size.txt` : '',
-      instanceSize ? `echo "Contents of /tmp/instance_size.txt: $(cat /tmp/instance_size.txt)"` : '',
+      `sudo mkdir -p ${persistentConfigDir}`,
+      `sudo chown root:root ${persistentConfigDir}`,
+      `sudo chmod 755 ${persistentConfigDir}`,
+      `echo "Writing service type '${service}' to ${persistentConfigDir}/service_type.txt"`,
+      `echo "${service}" | sudo tee ${persistentConfigDir}/service_type.txt`,
+      instanceSize ? `echo "Writing instance size '${instanceSize}' to ${persistentConfigDir}/instance_size.txt"` : '',
+      instanceSize ? `echo "${instanceSize}" | sudo tee ${persistentConfigDir}/instance_size.txt` : '',
       'sudo yum update -y',
       'sudo yum install -y amazon-cloudwatch-agent -y',
       'sudo dnf install -y wget ruby docker',
@@ -53,13 +53,13 @@ export default (
       'sudo usermod -a -G docker ec2-user',
       'sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose',
       'sudo chmod +x /usr/local/bin/docker-compose',
-      'docker-compose --version', // Verify installation
+      'docker-compose --version',
       'sudo yum install -y jq',
       `export SERVICE=${service}`,
       instanceSize ? `export INSTANCE_SIZE=${instanceSize}` : '',
       'exec 1>>/var/log/user-data.log 2>&1',
       'echo "Finished User Data Execution at $(date)"',
-      'sudo mkdir -p /etc/docker', // Ensure /etc/docker directory exists
+      'sudo mkdir -p /etc/docker',
 `cat << EOF | sudo tee /etc/docker/daemon.json
 {
   "log-driver": "awslogs",
@@ -69,10 +69,10 @@ export default (
     "awslogs-stream": "${service}"
   }
 }
-EOF`, // Ensure EOF is on a new line with no leading/trailing spaces
-    `sudo chmod 644 /etc/docker/daemon.json`, // Good practice to set permissions
+EOF`,
+    `sudo chmod 644 /etc/docker/daemon.json`,
     'sudo systemctl restart docker',
-      'sudo systemctl status docker'
+    'sudo systemctl status docker'
     );
     return ld;
   };

client-admin/package-lock.json

@@ -66,6 +66,7 @@
     "react-oidc-context": "^3.3.0",
     "react-redux": "^9.2.0",
     "react-router": "^7.6.3",
+    "react-router-dom": "^7.7.1",
     "redux": "^5.0.1",
     "theme-ui": "^0.17.2",
     "victory": "~37.3.6"

client-admin/package.json

@@ -11,6 +11,7 @@ import ConversationStats from './stats'
 import { useAuth } from 'react-oidc-context'
 
 import ModerateComments from './comment-moderation/'
+import TopicModeration from './topic-moderation/'
 
 // import DataExport from "./data-export";
 import ShareAndEmbed from './share-and-embed'
@@ -61,6 +62,16 @@ const ConversationAdminContainer = () => {
             All
           </Link>
         </Box>
+        <Box sx={{ mb: [3] }}>
+          <Link
+            sx={{
+              variant: url === 'topics' ? 'links.activeNav' : 'links.nav'
+            }}
+            data-test-id="moderate-topics"
+            to={`${baseUrl}/topics`}>
+            Topic Mod
+          </Link>
+        </Box>
         <Box sx={{ mb: [3] }}>
           <Link sx={{ variant: url ? 'links.nav' : 'links.activeNav' }} to={baseUrl}>
             Configure
@@ -111,6 +122,12 @@ const ConversationAdminContainer = () => {
           <Route path="reports/*" element={<Reports />} />
           <Route path="comments/*" element={<ModerateComments />} />
           <Route path="stats" element={<ConversationStats />} />
+          <Route
+            path={`${baseUrl}/topics`}
+            render={(props) => (
+              <TopicModeration {...props} conversation_id={params.conversation_id} />
+            )}
+          />
           {/* <Route path="export" element={<DataExport />} /> */}
         </Routes>
       </Box>

client-admin/src/components/conversation-admin/index.js

@@ -117,7 +117,7 @@ const ModerateCommentsSeed = ({ params }) => {
           Upload a CSV of seed comments
         </Heading>
         <input onChange={handleFileChange} type="file" id="csvFile" accept=".csv"></input>
-        <Button onClick={handleSubmitSeedBulk}>{getButtonText()}</Button>
+        <Button onClick={handleSubmitSeedBulk} data-testid="upload-csv-button">{getButtonText()}</Button>
       </Box>
     </Box>
   )

client-admin/src/components/conversation-admin/seed-comment.js

@@ -0,0 +1,66 @@
+# Topic Moderation Components
+
+This directory contains the React components for the Topic-Based Moderation system in pol.is.
+
+## Components
+
+### `index.js` - TopicModeration
+Main container component with tabbed navigation between different views:
+- Topics Tree view
+- Proximity Map visualization  
+- Statistics dashboard
+
+### `topic-tree.js` - TopicTree
+Hierarchical display of topics organized by layers:
+- Layer selection (0, 1, 2, or all)
+- Topic cards with moderation controls
+- Bulk topic-level actions (Accept/Reject/Meta)
+- Navigation to detailed comment view
+
+### `topic-detail.js` - TopicDetail  
+Detailed view of comments within a specific topic:
+- Individual comment display with selection
+- Bulk comment selection and actions
+- UMAP coordinate display
+- Moderation status tracking
+
+### `proximity-visualization.js` - ProximityVisualization
+SVG-based UMAP visualization:
+- Interactive scatter plot of comment positions
+- Cluster grouping visualization
+- Color coding by moderation status
+- Layer selection for different granularities
+
+### `topic-stats.js` - TopicStats
+Statistics and progress tracking:
+- Moderation completion rates
+- Status distribution (pending/accepted/rejected/meta)
+- Progress bars and visual indicators
+- Overview dashboard
+
+## Usage
+
+The components are integrated into the conversation admin interface at:
+`/m/:conversation_id/topics`
+
+## Dependencies
+
+- React with hooks
+- theme-ui for styling
+- React Router for navigation
+- SVG manipulation for visualizations
+
+## Data Flow
+
+1. Components fetch data from `/api/v3/topicMod/*` endpoints
+2. Real-time polling for updates (60-second intervals)
+3. Optimistic UI updates for moderation actions
+4. Error handling with retry mechanisms
+
+## Styling
+
+Uses theme-ui variants and custom CSS in `topic-moderation.css` for:
+- Hover effects and transitions
+- Status-based color coding
+- Responsive design
+- Loading states and animations