Feature: json RPC like at node (#424)

* Feature: json RPC like at node

* Fixes

Author: aopoltorzhicky

.github/workflows/build.yml

@@ -204,3 +204,49 @@ jobs:
           cache-to: type=gha,mode=max
           tags: ${{ steps.meta-celestials.outputs.tags }}
           labels: ${{ steps.meta-celestials.outputs.labels }}
+
+  # Json Rpc
+  build_jsonrpc:
+    name: Build Json RPC
+    runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      DOCKER_REGISTRY: ghcr.io
+      DOCKER_IMAGE_BASE: ${{ github.repository }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          
+      - name: Json RPC validate build configuration
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          call: check
+          file: build/jsonrpc/Dockerfile
+
+      - name: Json RPC image tags & labels
+        id: meta-jsonrpc
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_BASE }}-jsonrpc
+
+      - name: Json RPC image build & push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: build/jsonrpc/Dockerfile
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: ${{ steps.meta-jsonrpc.outputs.tags }}
+          labels: ${{ steps.meta-jsonrpc.outputs.labels }}

.vscode/launch.json

@@ -48,6 +48,17 @@
                 "../../configs/dipdup.yml",
             ],
             "envFile": "${workspaceFolder}/.env"
+        }, {
+            "name": "Launch JSON RPC",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceFolder}/cmd/jsonrpc",
+            "args": [
+                "-c",
+                "../../configs/dipdup.yml",
+            ],
+            "envFile": "${workspaceFolder}/.env"
         },
     ]
 }

Makefile

@@ -13,6 +13,9 @@ api:
 private_api:
 	cd cmd/private_api && go run . -c ../../configs/dipdup.yml
 
+jsonrpc:
+	cd cmd/jsonrpc && go run . -c ../../configs/dipdup.yml
+
 celestials:
 	cd cmd/celestials && go run . -c ../../configs/dipdup.yml
 

build/jsonrpc/Dockerfile

@@ -0,0 +1,37 @@
+# ---------------------------------------------------------------------
+#  The first stage container, for building the application
+# ---------------------------------------------------------------------
+FROM golang:1.24.4-alpine AS builder
+
+ENV CGO_ENABLED=0
+ENV GO111MODULE=on
+ENV GOOS=linux
+
+RUN apk --no-cache add ca-certificates
+
+RUN mkdir -p $GOPATH/src/github.com/celenium-io/celestia-indexer/
+
+COPY ./go.* $GOPATH/src/github.com/celenium-io/celestia-indexer/
+WORKDIR $GOPATH/src/github.com/celenium-io/celestia-indexer
+RUN go mod download
+
+COPY cmd/jsonrpc cmd/jsonrpc
+COPY internal internal
+COPY pkg pkg
+
+WORKDIR $GOPATH/src/github.com/celenium-io/celestia-indexer/cmd/jsonrpc/
+RUN go build -a -installsuffix cgo -o /go/bin/jsonrpc .
+
+# ---------------------------------------------------------------------
+#  The second stage container, for running the application
+# ---------------------------------------------------------------------
+FROM scratch
+
+WORKDIR /app/jsonrpc
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /go/bin/jsonrpc /go/bin/jsonrpc
+COPY ./configs/dipdup.yml ./
+COPY database database
+
+ENTRYPOINT ["/go/bin/jsonrpc", "-c", "dipdup.yml"]

cmd/api/init.go

@@ -210,7 +210,7 @@ func initEcho(cfg ApiConfig, env string) *echo.Echo {
 	e.Use(middleware.DecompressWithConfig(middleware.DecompressConfig{
 		Skipper: websocketSkipper,
 	}))
-	e.Use(middleware.BodyLimit("2M"))
+	e.Use(middleware.BodyLimit("9M"))
 	e.Use(middleware.CSRFWithConfig(
 		middleware.CSRFConfig{
 			Skipper: func(c echo.Context) bool {

cmd/jsonrpc/config.go

@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: 2025 PK Lab AG <celenium@baking-bad.org>
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	indexerConfig "github.com/celenium-io/celestia-indexer/pkg/indexer/config"
+	"github.com/dipdup-net/go-lib/config"
+)
+
+type Config struct {
+	*config.Config `yaml:",inline"`
+	LogLevel       string                `validate:"omitempty,oneof=debug trace info warn error fatal panic" yaml:"log_level"`
+	JsonRpcConfig  JsonRpcConfig         `validate:"required"                                                yaml:"jsonrpc"`
+	Indexer        indexerConfig.Indexer `validate:"required"                                                yaml:"indexer"`
+	Environment    string                `validate:"omitempty,oneof=development production"                  yaml:"environment"`
+}
+
+type JsonRpcConfig struct {
+	Bind           string  `validate:"required,hostname_port" yaml:"bind"`
+	RateLimit      float64 `validate:"omitempty,min=0"        yaml:"rate_limit"`
+	Prometheus     bool    `validate:"omitempty"              yaml:"prometheus"`
+	RequestTimeout int     `validate:"omitempty,min=1"        yaml:"request_timeout"`
+	BlobReceiver   string  `validate:"required"               yaml:"blob_receiver"`
+	SentryDsn      string  `validate:"omitempty"              yaml:"sentry_dsn"`
+}

cmd/jsonrpc/handler.go

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: 2025 PK Lab AG <celenium@baking-bad.org>
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"slices"
+	"sync"
+
+	"github.com/celenium-io/celestia-indexer/internal/storage"
+	"github.com/celenium-io/celestia-indexer/pkg/node"
+	nodeTypes "github.com/celenium-io/celestia-indexer/pkg/node/types"
+	"github.com/celenium-io/celestia-indexer/pkg/types"
+)
+
+type BlobHandler struct {
+	receiver   node.DalApi
+	logs       storage.IBlobLog
+	namespaces storage.INamespace
+}
+
+func NewBlobHandler(
+	receiver node.DalApi,
+	logs storage.IBlobLog,
+	namespaces storage.INamespace,
+) *BlobHandler {
+	return &BlobHandler{
+		receiver:   receiver,
+		logs:       logs,
+		namespaces: namespaces,
+	}
+}
+
+func (h *BlobHandler) Get(ctx context.Context, height types.Level, namespace string, commitment string) (nodeTypes.Blob, error) {
+	return h.receiver.Blob(ctx, height, namespace, commitment)
+}
+
+func (h *BlobHandler) GetAll(ctx context.Context, height types.Level, namespaces []string) ([]nodeTypes.Blob, error) {
+	var (
+		resultBlobs = make([][]nodeTypes.Blob, len(namespaces))
+		resultErr   = make([]error, len(namespaces))
+		wg          = new(sync.WaitGroup)
+	)
+	for i, namespace := range namespaces {
+		wg.Add(1)
+		go func(i int, namespace string) {
+			defer wg.Done()
+
+			hash, err := base64.StdEncoding.DecodeString(namespace)
+			if err != nil {
+				resultErr[i] = err
+				return
+			}
+			ns, err := h.namespaces.ByNamespaceIdAndVersion(ctx, hash[1:], hash[0])
+			if err != nil {
+				resultErr[i] = err
+				return
+			}
+
+			blobs, err := h.logs.ByNamespace(ctx, ns.Id, storage.BlobLogFilters{
+				Limit:  100,
+				Height: height,
+			})
+			if err != nil {
+				resultErr[i] = err
+				return
+			}
+			if len(blobs) > 0 {
+				res := make([]nodeTypes.Blob, 0)
+				for i := range blobs {
+					b, err := h.receiver.Blob(ctx, height, namespace, blobs[i].Commitment)
+					if err != nil {
+						resultErr[i] = err
+						return
+					}
+					res = append(res, b)
+				}
+				resultBlobs[i] = res
+			}
+		}(i, namespace)
+	}
+	wg.Wait()
+
+	blobs := slices.Concat(resultBlobs...)
+	err := errors.Join(resultErr...)
+	return blobs, err
+}

cmd/jsonrpc/handler_test.go

@@ -0,0 +1,112 @@
+// SPDX-FileCopyrightText: 2025 PK Lab AG <celenium@baking-bad.org>
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"context"
+	"testing"
+
+	"github.com/celenium-io/celestia-indexer/internal/storage"
+	"github.com/celenium-io/celestia-indexer/internal/storage/mock"
+	testsuite "github.com/celenium-io/celestia-indexer/internal/test_suite"
+	nodeMock "github.com/celenium-io/celestia-indexer/pkg/node/mock"
+	"github.com/celenium-io/celestia-indexer/pkg/node/types"
+	pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types"
+	"github.com/labstack/echo/v4"
+	"github.com/stretchr/testify/suite"
+	"go.uber.org/mock/gomock"
+)
+
+// HandlerTestSuite -
+type HandlerTestSuite struct {
+	suite.Suite
+	ns           *mock.MockINamespace
+	logs         *mock.MockIBlobLog
+	blobReceiver *nodeMock.MockDalApi
+	echo         *echo.Echo
+	handler      *BlobHandler
+	ctrl         *gomock.Controller
+}
+
+// SetupSuite -
+func (s *HandlerTestSuite) SetupSuite() {
+	s.echo = echo.New()
+	s.ctrl = gomock.NewController(s.T())
+	s.ns = mock.NewMockINamespace(s.ctrl)
+	s.logs = mock.NewMockIBlobLog(s.ctrl)
+	s.blobReceiver = nodeMock.NewMockDalApi(s.ctrl)
+	s.handler = NewBlobHandler(s.blobReceiver, s.logs, s.ns)
+}
+
+// TearDownSuite -
+func (s *HandlerTestSuite) TearDownSuite() {
+	s.ctrl.Finish()
+	s.Require().NoError(s.echo.Shutdown(context.Background()))
+}
+
+func TestSuiteHandler_Run(t *testing.T) {
+	suite.Run(t, new(HandlerTestSuite))
+}
+
+func (s *HandlerTestSuite) TestGet() {
+	data := testsuite.RandomText(100)
+	s.blobReceiver.EXPECT().
+		Blob(gomock.Any(), pkgTypes.Level(6357101), "AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=", "TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=").
+		Return(types.Blob{
+			Namespace:    "AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=",
+			Commitment:   "TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=",
+			ShareVersion: 0,
+			Data:         data,
+		}, nil).
+		Times(1)
+
+	response, err := s.handler.Get(s.T().Context(), 6357101, "AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=", "TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=")
+	s.Require().NoError(err)
+	s.Require().EqualValues("AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=", response.Namespace)
+	s.Require().EqualValues("TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=", response.Commitment)
+	s.Require().EqualValues(data, response.Data)
+	s.Require().EqualValues(0, response.ShareVersion)
+}
+
+func (s *HandlerTestSuite) TestGetAll() {
+	data := testsuite.RandomText(100)
+
+	s.ns.EXPECT().
+		ByNamespaceIdAndVersion(gomock.Any(), gomock.Any(), byte(0)).
+		Return(storage.Namespace{
+			Id: 1,
+		}, nil).
+		Times(1)
+
+	s.logs.EXPECT().
+		ByNamespace(gomock.Any(), uint64(1), storage.BlobLogFilters{
+			Limit:  100,
+			Height: 6357101,
+		}).
+		Return([]storage.BlobLog{
+			{
+				Commitment: "TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=",
+			},
+		}, nil).
+		Times(1)
+
+	s.blobReceiver.EXPECT().
+		Blob(gomock.Any(), pkgTypes.Level(6357101), "AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=", "TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=").
+		Return(types.Blob{
+			Namespace:    "AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=",
+			Commitment:   "TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=",
+			ShareVersion: 0,
+			Data:         data,
+		}, nil).
+		Times(1)
+
+	response, err := s.handler.GetAll(s.T().Context(), 6357101, []string{"AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8="})
+	s.Require().NoError(err)
+	s.Require().Len(response, 1)
+
+	s.Require().EqualValues("AAAAAAAAAAAAAAAAAAAAAAAAAMod4SqXFNgavI8=", response[0].Namespace)
+	s.Require().EqualValues("TekzHEK3JoBFapZdQFUoPwtuzUgpEd6OWyhGGzFTv3s=", response[0].Commitment)
+	s.Require().EqualValues(data, response[0].Data)
+	s.Require().EqualValues(0, response[0].ShareVersion)
+}