From 7ba62f873fffa091bc68dbff0d72c33a664f6b44 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:11:28 +0000 Subject: [PATCH 1/5] Initial plan From 12c84ade824c0cfdbbcdef3067445c4117bc266e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:19:01 +0000 Subject: [PATCH 2/5] Add scalawiki-duckdb module with Quill integration and ImageRepository Co-authored-by: intracer <232811+intracer@users.noreply.github.com> --- build.sbt | 17 +- project/Dependencies.scala | 10 + .../scalawiki/duckdb/ImageRepository.scala | 222 ++++++++++++++++++ .../duckdb/ImageRepositorySpec.scala | 208 ++++++++++++++++ 4 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala create mode 100644 scalawiki-duckdb/src/test/scala/org/scalawiki/duckdb/ImageRepositorySpec.scala diff --git a/build.sbt b/build.sbt index d5437b49..8fdcd3bc 100644 --- a/build.sbt +++ b/build.sbt @@ -47,8 +47,8 @@ lazy val commonSettings = Seq( lazy val scalawiki = (project in file(".")) .settings(commonSettings) - .dependsOn(core, bots, dumps, wlx, `http-extensions`) - .aggregate(core, bots, dumps, wlx, `http-extensions`) + .dependsOn(core, bots, dumps, wlx, duckdb, `http-extensions`) + .aggregate(core, bots, dumps, wlx, duckdb, `http-extensions`) lazy val core = Project("scalawiki-core", file("scalawiki-core")) .settings(commonSettings: _*) @@ -131,3 +131,16 @@ lazy val `http-extensions` = (project in file("http-extensions")) "org.scalacheck" %% "scalacheck" % ScalaCheckV % Test ) ) + +lazy val duckdb = Project("scalawiki-duckdb", file("scalawiki-duckdb")) + .dependsOn(core % "compile->compile;test->test") + .settings(commonSettings: _*) + .settings( + libraryDependencies ++= Seq( + Library.Quill.quillJdbc, + Library.DuckDb.jdbc, + Library.Specs2.core % Test, + Library.Specs2.matcherExtra % Test, + Library.Specs2.mock % Test + ) + ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2a63bc60..78a1f867 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -7,6 +7,7 @@ object Dependencies { val BlameApiV = "15.10.15" val ChronicleMapV = "3.27ea1" val ChronoScalaV = "1.0.0" + val DuckDbV = "1.1.3" val FicusV = "1.5.2" val GuavaV = "33.4.0-jre" val H2V = "1.4.200" @@ -15,6 +16,7 @@ object Dependencies { val JSoupV = "1.18.3" val LogbackClassicV = "1.5.15" val MockServerV = "5.15.0" + val QuillV = "4.8.6" val ReactiveStreamsV = "1.0.4" val RetryV = "0.3.6" val Scala213V = "2.13.15" @@ -135,6 +137,14 @@ object Dependencies { val mock = "org.specs2" %% "specs2-mock" % SpecsV } + object Quill { + val quillJdbc = "io.getquill" %% "quill-jdbc" % QuillV + } + + object DuckDb { + val jdbc = "org.duckdb" % "duckdb_jdbc" % DuckDbV + } + } } diff --git a/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala b/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala new file mode 100644 index 00000000..1b9e32da --- /dev/null +++ b/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala @@ -0,0 +1,222 @@ +package org.scalawiki.duckdb + +import io.getquill._ +import org.scalawiki.dto.Image +import java.time.ZonedDateTime +import javax.sql.DataSource + +/** + * Repository for reading and writing Image instances using Quill to DuckDB database + */ +class ImageRepository(dataSource: DataSource) { + + private val ctx = new PostgresJdbcContext(NamingStrategy(SnakeCase), dataSource) + import ctx._ + + /** + * Database schema for Image + */ + case class ImageRow( + title: String, + url: Option[String] = None, + pageUrl: Option[String] = None, + size: Option[Long] = None, + width: Option[Int] = None, + height: Option[Int] = None, + author: Option[String] = None, + uploaderLogin: Option[String] = None, + year: Option[String] = None, + date: Option[ZonedDateTime] = None, + monumentIds: String = "", + pageId: Option[Long] = None, + categories: String = "", + specialNominations: String = "", + mime: Option[String] = None + ) + + /** + * Convert Image DTO to database row + */ + private def toImageRow(image: Image): ImageRow = { + ImageRow( + title = image.title, + url = image.url, + pageUrl = image.pageUrl, + size = image.size, + width = image.width, + height = image.height, + author = image.author, + uploaderLogin = image.uploader.flatMap(_.login), + year = image.year, + date = image.date, + monumentIds = image.monumentIds.mkString(","), + pageId = image.pageId, + categories = image.categories.mkString(","), + specialNominations = image.specialNominations.mkString(","), + mime = image.mime + ) + } + + /** + * Convert database row to Image DTO + */ + private def fromImageRow(row: ImageRow): Image = { + Image( + title = row.title, + url = row.url, + pageUrl = row.pageUrl, + size = row.size, + width = row.width, + height = row.height, + author = row.author, + uploader = row.uploaderLogin.map(login => org.scalawiki.dto.User(None, Some(login))), + year = row.year, + date = row.date, + monumentIds = if (row.monumentIds.isEmpty) Seq.empty else row.monumentIds.split(",").toSeq, + pageId = row.pageId, + metadata = None, + categories = if (row.categories.isEmpty) Set.empty else row.categories.split(",").toSet, + specialNominations = if (row.specialNominations.isEmpty) Set.empty else row.specialNominations.split(",").toSet, + mime = row.mime + ) + } + + /** + * Insert an Image into the database + */ + def insert(image: Image): Long = { + val row = toImageRow(image) + ctx.run(query[ImageRow].insertValue(lift(row))) + } + + /** + * Insert multiple Images into the database + */ + def insertBatch(images: Seq[Image]): List[Long] = { + val rows = images.map(toImageRow) + ctx.run(liftQuery(rows).foreach(row => query[ImageRow].insertValue(row))) + } + + /** + * Find an Image by title + */ + def findByTitle(title: String): Option[Image] = { + ctx.run(query[ImageRow].filter(_.title == lift(title))).headOption.map(fromImageRow) + } + + /** + * Find all Images + */ + def findAll(): Seq[Image] = { + ctx.run(query[ImageRow]).map(fromImageRow) + } + + /** + * Find Images by monument ID + */ + def findByMonumentId(monumentId: String): Seq[Image] = { + ctx.run(query[ImageRow].filter(row => + sql"${row.monumentIds} LIKE ${"%" + monumentId + "%"}".asCondition + )).map(fromImageRow) + } + + /** + * Find Images by author + */ + def findByAuthor(author: String): Seq[Image] = { + ctx.run(query[ImageRow].filter(_.author.exists(_ == lift(author)))).map(fromImageRow) + } + + /** + * Update an Image + */ + def update(image: Image): Long = { + val row = toImageRow(image) + ctx.run(query[ImageRow].filter(_.title == lift(row.title)).updateValue(lift(row))) + } + + /** + * Delete an Image by title + */ + def delete(title: String): Long = { + ctx.run(query[ImageRow].filter(_.title == lift(title)).delete) + } + + /** + * Count all Images + */ + def count(): Long = { + ctx.run(query[ImageRow].size) + } + + /** + * Create the images table if it doesn't exist + */ + def createTable(): Unit = { + val createTableSql = """ + CREATE TABLE IF NOT EXISTS image_row ( + title VARCHAR PRIMARY KEY, + url VARCHAR, + page_url VARCHAR, + size BIGINT, + width INTEGER, + height INTEGER, + author VARCHAR, + uploader_login VARCHAR, + year VARCHAR, + date TIMESTAMP WITH TIME ZONE, + monument_ids VARCHAR, + page_id BIGINT, + categories VARCHAR, + special_nominations VARCHAR, + mime VARCHAR + ) + """ + val connection = dataSource.getConnection() + try { + val statement = connection.createStatement() + try { + statement.execute(createTableSql) + } finally { + statement.close() + } + } finally { + connection.close() + } + } + + /** + * Drop the images table + */ + def dropTable(): Unit = { + val connection = dataSource.getConnection() + try { + val statement = connection.createStatement() + try { + statement.execute("DROP TABLE IF EXISTS image_row") + } finally { + statement.close() + } + } finally { + connection.close() + } + } +} + +object ImageRepository { + /** + * Create a new ImageRepository with a DuckDB connection + */ + def apply(jdbcUrl: String): ImageRepository = { + val ds = new org.duckdb.DuckDBDataSource() + ds.setUrl(jdbcUrl) + new ImageRepository(ds) + } + + /** + * Create a new ImageRepository with an in-memory DuckDB database + */ + def inMemory(): ImageRepository = { + apply("jdbc:duckdb:") + } +} diff --git a/scalawiki-duckdb/src/test/scala/org/scalawiki/duckdb/ImageRepositorySpec.scala b/scalawiki-duckdb/src/test/scala/org/scalawiki/duckdb/ImageRepositorySpec.scala new file mode 100644 index 00000000..0b06efba --- /dev/null +++ b/scalawiki-duckdb/src/test/scala/org/scalawiki/duckdb/ImageRepositorySpec.scala @@ -0,0 +1,208 @@ +package org.scalawiki.duckdb + +import org.scalawiki.dto.{Image, User} +import org.specs2.mutable.Specification +import org.specs2.specification.BeforeAfterEach +import java.time.{ZonedDateTime, ZoneOffset} + +class ImageRepositorySpec extends Specification with BeforeAfterEach { + + sequential + + var repository: ImageRepository = _ + + override def before: Any = { + repository = ImageRepository.inMemory() + repository.createTable() + } + + override def after: Any = { + repository.dropTable() + } + + "ImageRepository" should { + + "insert and retrieve an image" in { + val image = Image( + title = "Test_Image.jpg", + url = Some("http://example.com/test.jpg"), + pageUrl = Some("http://example.com/page"), + size = Some(1024L), + width = Some(800), + height = Some(600), + author = Some("TestAuthor"), + uploader = Some(User(Some(1L), Some("TestUploader"))), + year = Some("2023"), + monumentIds = Seq("ID123", "ID456"), + pageId = Some(42L), + categories = Set("Category1", "Category2"), + specialNominations = Set("Nomination1"), + mime = Some("image/jpeg") + ) + + repository.insert(image) + + val retrieved = repository.findByTitle("Test_Image.jpg") + retrieved must beSome + retrieved.get.title must beEqualTo(image.title) + retrieved.get.url must beEqualTo(image.url) + retrieved.get.author must beEqualTo(image.author) + retrieved.get.uploader.flatMap(_.login) must beEqualTo(Some("TestUploader")) + retrieved.get.monumentIds must beEqualTo(image.monumentIds) + retrieved.get.categories must beEqualTo(image.categories) + retrieved.get.specialNominations must beEqualTo(image.specialNominations) + } + + "insert and retrieve multiple images" in { + val image1 = Image( + title = "Image1.jpg", + author = Some("Author1"), + monumentIds = Seq("MON1") + ) + + val image2 = Image( + title = "Image2.jpg", + author = Some("Author2"), + monumentIds = Seq("MON2") + ) + + repository.insertBatch(Seq(image1, image2)) + + val all = repository.findAll() + all must have size 2 + all.map(_.title) must contain(exactly("Image1.jpg", "Image2.jpg")) + } + + "find images by author" in { + val image1 = Image( + title = "Image1.jpg", + author = Some("AuthorA") + ) + + val image2 = Image( + title = "Image2.jpg", + author = Some("AuthorB") + ) + + val image3 = Image( + title = "Image3.jpg", + author = Some("AuthorA") + ) + + repository.insertBatch(Seq(image1, image2, image3)) + + val byAuthorA = repository.findByAuthor("AuthorA") + byAuthorA must have size 2 + byAuthorA.map(_.title) must contain(exactly("Image1.jpg", "Image3.jpg")) + } + + "find images by monument ID" in { + val image1 = Image( + title = "Image1.jpg", + monumentIds = Seq("MON123") + ) + + val image2 = Image( + title = "Image2.jpg", + monumentIds = Seq("MON456") + ) + + val image3 = Image( + title = "Image3.jpg", + monumentIds = Seq("MON123", "MON789") + ) + + repository.insertBatch(Seq(image1, image2, image3)) + + val byMonument = repository.findByMonumentId("MON123") + byMonument must have size 2 + byMonument.map(_.title) must contain(exactly("Image1.jpg", "Image3.jpg")) + } + + "update an existing image" in { + val image = Image( + title = "UpdateTest.jpg", + author = Some("OriginalAuthor") + ) + + repository.insert(image) + + val updatedImage = image.copy(author = Some("UpdatedAuthor")) + repository.update(updatedImage) + + val retrieved = repository.findByTitle("UpdateTest.jpg") + retrieved must beSome + retrieved.get.author must beEqualTo(Some("UpdatedAuthor")) + } + + "delete an image" in { + val image = Image( + title = "DeleteTest.jpg", + author = Some("TestAuthor") + ) + + repository.insert(image) + repository.findByTitle("DeleteTest.jpg") must beSome + + repository.delete("DeleteTest.jpg") + repository.findByTitle("DeleteTest.jpg") must beNone + } + + "count images" in { + val image1 = Image(title = "Image1.jpg") + val image2 = Image(title = "Image2.jpg") + val image3 = Image(title = "Image3.jpg") + + repository.insertBatch(Seq(image1, image2, image3)) + + repository.count() must beEqualTo(3L) + } + + "handle images with dates" in { + val date = ZonedDateTime.of(2023, 6, 15, 12, 30, 0, 0, ZoneOffset.UTC) + val image = Image( + title = "DateTest.jpg", + date = Some(date) + ) + + repository.insert(image) + + val retrieved = repository.findByTitle("DateTest.jpg") + retrieved must beSome + // Note: DuckDB timestamp comparison may have precision differences + retrieved.get.date.map(_.toEpochSecond) must beEqualTo(Some(date.toEpochSecond)) + } + + "handle empty monument IDs and categories" in { + val image = Image( + title = "EmptyTest.jpg", + monumentIds = Seq.empty, + categories = Set.empty, + specialNominations = Set.empty + ) + + repository.insert(image) + + val retrieved = repository.findByTitle("EmptyTest.jpg") + retrieved must beSome + retrieved.get.monumentIds must beEmpty + retrieved.get.categories must beEmpty + retrieved.get.specialNominations must beEmpty + } + + "return empty result when image not found" in { + repository.findByTitle("NonExistent.jpg") must beNone + } + + "return empty list when no images match criteria" in { + val image = Image( + title = "Test.jpg", + author = Some("AuthorA") + ) + repository.insert(image) + + repository.findByAuthor("NonExistentAuthor") must beEmpty + repository.findByMonumentId("NonExistentMonument") must beEmpty + } + } +} From b1ee1341b1c04a865ee78e204a049a668e7ff149 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:20:39 +0000 Subject: [PATCH 3/5] Add documentation and clarify DuckDB PostgreSQL compatibility Co-authored-by: intracer <232811+intracer@users.noreply.github.com> --- scalawiki-duckdb/.gitignore | 3 + scalawiki-duckdb/README.md | 91 +++++++++++++++++++ .../scalawiki/duckdb/ImageRepository.scala | 1 + 3 files changed, 95 insertions(+) create mode 100644 scalawiki-duckdb/.gitignore create mode 100644 scalawiki-duckdb/README.md diff --git a/scalawiki-duckdb/.gitignore b/scalawiki-duckdb/.gitignore new file mode 100644 index 00000000..101299a9 --- /dev/null +++ b/scalawiki-duckdb/.gitignore @@ -0,0 +1,3 @@ +*.db +*.db.wal +target/ diff --git a/scalawiki-duckdb/README.md b/scalawiki-duckdb/README.md new file mode 100644 index 00000000..a21c62a5 --- /dev/null +++ b/scalawiki-duckdb/README.md @@ -0,0 +1,91 @@ +# ScalaWiki DuckDB Module + +This module provides DuckDB database integration for ScalaWiki using Quill. + +## Features + +- **ImageRepository**: Repository for reading and writing `org.scalawiki.dto.Image` instances +- **Quill Integration**: Uses Quill for type-safe database queries +- **DuckDB Support**: Leverages DuckDB's embedded database capabilities + +## Usage + +### Creating a Repository + +```scala +import org.scalawiki.duckdb.ImageRepository + +// Create an in-memory repository +val repository = ImageRepository.inMemory() + +// Or create a file-based repository +val fileRepository = ImageRepository("jdbc:duckdb:/path/to/database.db") + +// Initialize the database table +repository.createTable() +``` + +### CRUD Operations + +```scala +import org.scalawiki.dto.Image + +// Insert an image +val image = Image( + title = "Example.jpg", + url = Some("http://example.com/image.jpg"), + author = Some("John Doe"), + monumentIds = Seq("MON123") +) +repository.insert(image) + +// Find by title +val found = repository.findByTitle("Example.jpg") + +// Find by author +val byAuthor = repository.findByAuthor("John Doe") + +// Find by monument ID +val byMonument = repository.findByMonumentId("MON123") + +// Update +val updated = image.copy(author = Some("Jane Doe")) +repository.update(updated) + +// Delete +repository.delete("Example.jpg") + +// Get all images +val all = repository.findAll() + +// Count images +val count = repository.count() +``` + +### Batch Operations + +```scala +// Insert multiple images +val images = Seq( + Image(title = "Image1.jpg"), + Image(title = "Image2.jpg"), + Image(title = "Image3.jpg") +) +repository.insertBatch(images) +``` + +## Dependencies + +This module depends on: +- `io.getquill:quill-jdbc:4.8.6` - Quill JDBC support +- `org.duckdb:duckdb_jdbc:1.1.3` - DuckDB JDBC driver +- `scalawiki-core` - Core ScalaWiki DTOs + +## Testing + +Run tests with: +``` +sbt "project scalawiki-duckdb" test +``` + +All tests use in-memory DuckDB databases for fast, isolated test execution. diff --git a/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala b/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala index 1b9e32da..b89e7e5d 100644 --- a/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala +++ b/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala @@ -10,6 +10,7 @@ import javax.sql.DataSource */ class ImageRepository(dataSource: DataSource) { + // DuckDB is compatible with PostgreSQL dialect for most operations private val ctx = new PostgresJdbcContext(NamingStrategy(SnakeCase), dataSource) import ctx._ From 5aa6071fafd7843b566659dbdecd5ec0062283a5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:21:28 +0000 Subject: [PATCH 4/5] Fix SQL injection vulnerability in findByMonumentId query Co-authored-by: intracer <232811+intracer@users.noreply.github.com> --- .../src/main/scala/org/scalawiki/duckdb/ImageRepository.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala b/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala index b89e7e5d..b7aa76dc 100644 --- a/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala +++ b/scalawiki-duckdb/src/main/scala/org/scalawiki/duckdb/ImageRepository.scala @@ -116,8 +116,9 @@ class ImageRepository(dataSource: DataSource) { * Find Images by monument ID */ def findByMonumentId(monumentId: String): Seq[Image] = { + val pattern = s"%$monumentId%" ctx.run(query[ImageRow].filter(row => - sql"${row.monumentIds} LIKE ${"%" + monumentId + "%"}".asCondition + sql"${row.monumentIds} LIKE ${lift(pattern)}".asCondition )).map(fromImageRow) } From abdd77accb9e4eded5deb4af70c2a8502c5fcda3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 10:22:58 +0000 Subject: [PATCH 5/5] Enhance DuckDB module documentation with security features --- scalawiki-duckdb/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scalawiki-duckdb/README.md b/scalawiki-duckdb/README.md index a21c62a5..c07e7b3d 100644 --- a/scalawiki-duckdb/README.md +++ b/scalawiki-duckdb/README.md @@ -1,12 +1,13 @@ # ScalaWiki DuckDB Module -This module provides DuckDB database integration for ScalaWiki using Quill. +This module provides DuckDB database integration for ScalaWiki using Quill for type-safe, compile-time checked database queries. ## Features - **ImageRepository**: Repository for reading and writing `org.scalawiki.dto.Image` instances -- **Quill Integration**: Uses Quill for type-safe database queries -- **DuckDB Support**: Leverages DuckDB's embedded database capabilities +- **Quill Integration**: Uses Quill for type-safe database queries with compile-time verification +- **DuckDB Support**: Leverages DuckDB's embedded database capabilities for fast, serverless analytics +- **SQL Injection Protection**: All queries use parameterized statements via Quill's lift mechanism ## Usage