From 3ee5d175e28ecae595892567611e659bf76fb2a2 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 2 Sep 2025 14:55:36 -0700 Subject: [PATCH 01/27] setup etc/hosts in roofts and add post-routing rule to catch requests to events domain --- packages/orchestrator/internal/consts.go | 13 +++++++++++++ .../internal/sandbox/network/network.go | 8 ++++++++ .../internal/template/build/core/rootfs/rootfs.go | 6 +++++- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 packages/orchestrator/internal/consts.go diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go new file mode 100644 index 0000000000..2d0e30657c --- /dev/null +++ b/packages/orchestrator/internal/consts.go @@ -0,0 +1,13 @@ +package internal + +import ( + "github.com/e2b-dev/infra/packages/shared/pkg/env" +) + +const ( + defaultSandboxEventIP = "203.0.113.0" +) + +func GetSandboxEventIP() string { + return env.GetEnv("SANDBOX_EVENT_IP", defaultSandboxEventIP) +} diff --git a/packages/orchestrator/internal/sandbox/network/network.go b/packages/orchestrator/internal/sandbox/network/network.go index 6fadb92ed2..d7746395d5 100644 --- a/packages/orchestrator/internal/sandbox/network/network.go +++ b/packages/orchestrator/internal/sandbox/network/network.go @@ -7,6 +7,7 @@ import ( "runtime" "github.com/coreos/go-iptables/iptables" + "github.com/e2b-dev/infra/packages/orchestrator/internal" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "go.uber.org/zap" @@ -214,6 +215,13 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } + // Redirect traffic destined for event server + eventIP := internal.GetSandboxEventIP() + err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") + if err != nil { + return fmt.Errorf("error creating HTTP redirect rule to sandbox event server: %w", err) + } + return nil } diff --git a/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go b/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go index 6b296ba360..7e1b0237ac 100644 --- a/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go +++ b/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/orchestrator/internal" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/config" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/filesystem" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/oci" @@ -199,6 +200,8 @@ ExecStart=-/sbin/agetty --noissue --autologin root %I 115200,38400,9600 vt102 ` hostname := "e2b.local" + eventProxyHostname := "events.e2b.dev" + eventIP := internal.GetSandboxEventIP() hosts := fmt.Sprintf(`127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -207,7 +210,8 @@ ff00:: ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters 127.0.1.1 %s -`, hostname) +%s %s +`, hostname, eventIP, eventProxyHostname) envdFileData, err := os.ReadFile(storage.HostEnvdPath) if err != nil { From 55f0854a0c34ae8844c2a4593c47c6afe8902743 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 2 Sep 2025 15:01:09 -0700 Subject: [PATCH 02/27] add comment explaining why we use a reserved IP address --- packages/orchestrator/internal/consts.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go index 2d0e30657c..783d824e9f 100644 --- a/packages/orchestrator/internal/consts.go +++ b/packages/orchestrator/internal/consts.go @@ -5,6 +5,8 @@ import ( ) const ( + // Private IP don't leave the sandbox through the network bridge, so we use a reserved IP address for it: + // See TEST-NET-3 on https://en.wikipedia.org/wiki/Reserved_IP_addresses defaultSandboxEventIP = "203.0.113.0" ) From 423fb4bbcb622316d5d7df81275d46b7d26c84ee Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 2 Sep 2025 15:16:31 -0700 Subject: [PATCH 03/27] golangci-lint --- packages/orchestrator/internal/sandbox/network/network.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/internal/sandbox/network/network.go b/packages/orchestrator/internal/sandbox/network/network.go index d7746395d5..ec9d5459d8 100644 --- a/packages/orchestrator/internal/sandbox/network/network.go +++ b/packages/orchestrator/internal/sandbox/network/network.go @@ -7,10 +7,11 @@ import ( "runtime" "github.com/coreos/go-iptables/iptables" - "github.com/e2b-dev/infra/packages/orchestrator/internal" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/orchestrator/internal" ) func (s *Slot) CreateNetwork() error { From cbc73135df4b72094d312c24497d3f501a353f73 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 2 Sep 2025 17:30:47 -0700 Subject: [PATCH 04/27] * inatroduce sandbox event proxy & handlers * map sandbox IPs to sandbox IDs --- .../events/sandbox_events_handlers.go | 102 ++++++++++++++ .../internal/events/sandbox_events_proxy.go | 48 +++++++ .../internal/events/sandbox_events_store.go | 133 ++++++++++++++++++ .../orchestrator/internal/sandbox/sandbox.go | 8 ++ packages/orchestrator/internal/server/main.go | 3 + .../orchestrator/internal/server/sandboxes.go | 1 + .../template/build/layer/create_sandbox.go | 2 + .../template/build/layer/interfaces.go | 2 + .../template/build/layer/layer_executor.go | 2 +- .../template/build/layer/resume_sandbox.go | 3 + packages/orchestrator/main.go | 18 ++- 11 files changed, 317 insertions(+), 5 deletions(-) create mode 100644 packages/orchestrator/internal/events/sandbox_events_handlers.go create mode 100644 packages/orchestrator/internal/events/sandbox_events_proxy.go create mode 100644 packages/orchestrator/internal/events/sandbox_events_store.go diff --git a/packages/orchestrator/internal/events/sandbox_events_handlers.go b/packages/orchestrator/internal/events/sandbox_events_handlers.go new file mode 100644 index 0000000000..86694faa53 --- /dev/null +++ b/packages/orchestrator/internal/events/sandbox_events_handlers.go @@ -0,0 +1,102 @@ +package events + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + + "go.uber.org/zap" +) + +type SandboxEventHandler interface { + Path() string + HandlerFunc(w http.ResponseWriter, r *http.Request) +} + +// This is used to track ad-hoc events that are not handled by the event server. +type DefaultSandboxEventHandler struct { + store SandboxEventStore +} + +func (h *DefaultSandboxEventHandler) Path() string { + return "/" +} + +func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { + addr := r.RemoteAddr + ip := strings.Split(addr, ":")[0] + sandboxID, err := h.store.GetSandboxIP(ip) + if err != nil { + zap.L().Error("Failed to get sandbox ID from IP", zap.Error(err)) + http.Error(w, "Error handling event", http.StatusInternalServerError) + return + } + + zap.L().Debug("Received request from sandbox", zap.String("sandbox_id", sandboxID), zap.String("ip", ip)) + + if r.Method == http.MethodGet { + events, err := h.store.GetLastNEvents(sandboxID, 10) + if err != nil { + zap.L().Error("Failed to get event data for sandbox "+sandboxID, zap.Error(err)) + http.Error(w, "Failed to get event data for sandbox "+sandboxID, http.StatusInternalServerError) + return + } + + eventJSON, err := json.Marshal(events) + if err != nil { + zap.L().Error("Failed to marshal event data", zap.Error(err)) + http.Error(w, "Failed to marshal event data", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(eventJSON) + return + } + + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Create event data with path and body + eventData := SandboxEvent{ + Path: r.URL.Path, + } + + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusInternalServerError) + return + } + + zap.L().Info("Received event", zap.String("body", string(body))) + + eventData.Body = make(map[string]any) + err = json.Unmarshal(body, &eventData.Body) + if err != nil { + zap.L().Error("Failed to unmarshal request body", zap.Error(err)) + http.Error(w, "Failed to unmarshal request body", http.StatusInternalServerError) + return + } + + // Store in Redis with sandboxID as key + err = h.store.AddEvent(sandboxID, &eventData, 0) + if err != nil { + zap.L().Error("Failed to store event data", zap.Error(err)) + http.Error(w, "Failed to store event data", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"event_ack":true}`)) +} + +func NewSandboxEventHandlers(ctx context.Context, store SandboxEventStore) []SandboxEventHandler { + return []SandboxEventHandler{ + &DefaultSandboxEventHandler{store}, + } +} diff --git a/packages/orchestrator/internal/events/sandbox_events_proxy.go b/packages/orchestrator/internal/events/sandbox_events_proxy.go new file mode 100644 index 0000000000..8e0a7ba953 --- /dev/null +++ b/packages/orchestrator/internal/events/sandbox_events_proxy.go @@ -0,0 +1,48 @@ +package events + +import ( + "context" + "fmt" + "net/http" +) + +// SandboxEventServer handles outbound HTTP requests from sandboxes calling the event.e2b.com endpoint +type SandboxEventProxy struct { + server *http.Server +} + +func NewSandboxEventProxy(port uint, handlers []SandboxEventHandler) *SandboxEventProxy { + mux := http.NewServeMux() + + for _, handler := range handlers { + mux.HandleFunc(handler.Path(), handler.HandlerFunc) + } + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: mux, + } + + return &SandboxEventProxy{ + server: server, + } +} + +func (p *SandboxEventProxy) Start() error { + return p.server.ListenAndServe() +} + +func (p *SandboxEventProxy) Close(ctx context.Context) error { + var err error + select { + case <-ctx.Done(): + err = p.server.Close() + default: + err = p.server.Shutdown(ctx) + } + if err != nil { + return fmt.Errorf("failed to shutdown event server: %w", err) + } + + return nil +} diff --git a/packages/orchestrator/internal/events/sandbox_events_store.go b/packages/orchestrator/internal/events/sandbox_events_store.go new file mode 100644 index 0000000000..4523da3a68 --- /dev/null +++ b/packages/orchestrator/internal/events/sandbox_events_store.go @@ -0,0 +1,133 @@ +package events + +import ( + "context" + "encoding/json" + "time" + + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/trace" +) + +const ( + cacheTL = time.Hour * 24 * 30 + + EventPrefix = "ev:" + IPPrefix = "ip:" +) + +type SandboxEvent struct { + Path string `json:"path"` + Body map[string]any `json:"body"` +} + +func (i SandboxEvent) MarshalBinary() ([]byte, error) { + return json.Marshal(i) +} + +type sandboxEventStore struct { + ctx context.Context + tracer trace.Tracer + redisClient redis.UniversalClient +} + +type SandboxEventStore interface { + SetSandboxIP(sandboxId string, ip string) error + GetSandboxIP(sandboxId string) (string, error) + DelSandboxIP(sandboxId string) error + + GetLastEvent(sandboxId string) (*SandboxEvent, error) + GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) + AddEvent(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error + DelEvent(sandboxId string) error + + Close(ctx context.Context) error +} + +func NewSandboxEventStore(ctx context.Context, tracer trace.Tracer, redisClient redis.UniversalClient) SandboxEventStore { + return &sandboxEventStore{ + ctx: ctx, + tracer: tracer, + redisClient: redisClient, + } +} + +func (c *sandboxEventStore) SetSandboxIP(sandboxId string, ip string) error { + return c.redisClient.Set(c.ctx, IPPrefix+ip, sandboxId, cacheTL).Err() +} + +func (c *sandboxEventStore) GetSandboxIP(ip string) (string, error) { + return c.redisClient.Get(c.ctx, IPPrefix+ip).Result() +} + +func (c *sandboxEventStore) DelSandboxIP(ip string) error { + return c.redisClient.Del(c.ctx, IPPrefix+ip).Err() +} + +func (c *sandboxEventStore) GetLastEvent(sandboxId string) (*SandboxEvent, error) { + _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last") + defer span.End() + + result, err := c.redisClient.ZRevRangeWithScores(c.ctx, EventPrefix+sandboxId, 0, 0).Result() + if err != nil { + return nil, err + } + if len(result) == 0 { + return nil, redis.Nil + } + rawEvent := result[0].Member.(string) + + var event SandboxEvent + err = json.Unmarshal([]byte(rawEvent), &event) + if err != nil { + return nil, err + } + return &event, nil +} + +func (c *sandboxEventStore) GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) { + _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last-n") + defer span.End() + + result, err := c.redisClient.ZRevRangeWithScores(c.ctx, EventPrefix+sandboxId, 0, int64(n-1)).Result() + if err != nil { + return nil, err + } + if len(result) == 0 { + return nil, redis.Nil + } + + events := make([]*SandboxEvent, 0, len(result)) + for _, item := range result { + rawEvent := item.Member.(string) + var event SandboxEvent + err = json.Unmarshal([]byte(rawEvent), &event) + if err != nil { + return nil, err + } + events = append(events, &event) + } + + return events, nil +} + +func (c *sandboxEventStore) AddEvent(sandboxId string, event *SandboxEvent, expiration time.Duration) error { + _, span := c.tracer.Start(c.ctx, "sandbox-event-store") + defer span.End() + + return c.redisClient.ZAdd(c.ctx, EventPrefix+sandboxId, redis.Z{ + Score: float64(time.Now().UnixNano()), + Member: event, + }).Err() +} + +func (c *sandboxEventStore) DelEvent(sandboxId string) error { + _, span := c.tracer.Start(c.ctx, "sandbox-event-delete") + defer span.End() + + return c.redisClient.Del(c.ctx, EventPrefix+sandboxId).Err() +} + +func (c *sandboxEventStore) Close(ctx context.Context) error { + return c.redisClient.Close() +} diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index a1ac4823a9..53c2c90e19 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -14,6 +14,7 @@ import ( "go.uber.org/zap" globalconfig "github.com/e2b-dev/infra/packages/orchestrator/internal/config" + "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/fc" @@ -318,6 +319,7 @@ func ResumeSandbox( devicePool *nbd.DevicePool, useClickhouseMetrics bool, apiConfigToStore *orchestrator.SandboxConfig, + eventStore events.SandboxEventStore, ) (s *Sandbox, e error) { childCtx, childSpan := tracer.Start(ctx, "resume-sandbox") defer childSpan.End() @@ -518,6 +520,12 @@ func ResumeSandbox( return nil, fmt.Errorf("failed to wait for sandbox start: %w", err) } + sandboxIP := ips.slot.HostIPString() + eventStore.SetSandboxIP(runtime.SandboxID, sandboxIP) + cleanup.AddPriority(func(ctx context.Context) error { + return eventStore.DelSandboxIP(sandboxIP) + }) + go sbx.Checks.Start() go func() { diff --git a/packages/orchestrator/internal/server/main.go b/packages/orchestrator/internal/server/main.go index 41b56f58fb..5709d229ab 100644 --- a/packages/orchestrator/internal/server/main.go +++ b/packages/orchestrator/internal/server/main.go @@ -40,6 +40,7 @@ type server struct { featureFlags *featureflags.Client sbxEventsService events.EventsService[event.SandboxEvent] startingSandboxes *semaphore.Weighted + sbxEventsStore events.SandboxEventStore } type Service struct { @@ -68,6 +69,7 @@ type ServiceConfig struct { Persistence storage.StorageProvider FeatureFlags *featureflags.Client SbxEventsService events.EventsService[event.SandboxEvent] + SbxEventsStore events.SandboxEventStore } func New( @@ -91,6 +93,7 @@ func New( featureFlags: cfg.FeatureFlags, sbxEventsService: cfg.SbxEventsService, startingSandboxes: semaphore.NewWeighted(maxStartingInstancesPerNode), + sbxEventsStore: cfg.SbxEventsStore, } meter := cfg.Tel.MeterProvider.Meter("orchestrator.sandbox") diff --git a/packages/orchestrator/internal/server/sandboxes.go b/packages/orchestrator/internal/server/sandboxes.go index cc605d2ae5..47e1902a0b 100644 --- a/packages/orchestrator/internal/server/sandboxes.go +++ b/packages/orchestrator/internal/server/sandboxes.go @@ -121,6 +121,7 @@ func (s *server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ s.devicePool, metricsWriteFlag, req.Sandbox, + s.sbxEventsStore, ) if err != nil { err := errors.Join(err, context.Cause(ctx)) diff --git a/packages/orchestrator/internal/template/build/layer/create_sandbox.go b/packages/orchestrator/internal/template/build/layer/create_sandbox.go index 919c23482c..70ac220149 100644 --- a/packages/orchestrator/internal/template/build/layer/create_sandbox.go +++ b/packages/orchestrator/internal/template/build/layer/create_sandbox.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" + "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/fc" @@ -32,6 +33,7 @@ func (f *CreateSandbox) Sandbox( ctx context.Context, layerExecutor *LayerExecutor, sourceTemplate sbxtemplate.Template, + eventStore events.SandboxEventStore, ) (*sandbox.Sandbox, error) { // Create new memfile with the size of the sandbox RAM, this updates the underlying memfile. // This is ok as the sandbox is started from the beginning. diff --git a/packages/orchestrator/internal/template/build/layer/interfaces.go b/packages/orchestrator/internal/template/build/layer/interfaces.go index e46576edad..38b1621fa8 100644 --- a/packages/orchestrator/internal/template/build/layer/interfaces.go +++ b/packages/orchestrator/internal/template/build/layer/interfaces.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/template" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/metadata" @@ -21,6 +22,7 @@ type SandboxCreator interface { ctx context.Context, layerExecutor *LayerExecutor, template sbxtemplate.Template, + eventStore events.SandboxEventStore, ) (*sandbox.Sandbox, error) } diff --git a/packages/orchestrator/internal/template/build/layer/layer_executor.go b/packages/orchestrator/internal/template/build/layer/layer_executor.go index fc2caab98a..9760be0feb 100644 --- a/packages/orchestrator/internal/template/build/layer/layer_executor.go +++ b/packages/orchestrator/internal/template/build/layer/layer_executor.go @@ -88,7 +88,7 @@ func (lb *LayerExecutor) BuildLayer( } // Create or resume sandbox - sbx, err := cmd.SandboxCreator.Sandbox(ctx, lb, localTemplate) + sbx, err := cmd.SandboxCreator.Sandbox(ctx, lb, localTemplate, lb.sbxEventsStore) if err != nil { return metadata.Template{}, err } diff --git a/packages/orchestrator/internal/template/build/layer/resume_sandbox.go b/packages/orchestrator/internal/template/build/layer/resume_sandbox.go index fc33f08912..8654a069e6 100644 --- a/packages/orchestrator/internal/template/build/layer/resume_sandbox.go +++ b/packages/orchestrator/internal/template/build/layer/resume_sandbox.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" + "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/template" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/config" @@ -28,6 +29,7 @@ func (f *ResumeSandbox) Sandbox( ctx context.Context, layerExecutor *LayerExecutor, template sbxtemplate.Template, + eventStore events.SandboxEventStore, ) (*sandbox.Sandbox, error) { sbx, err := sandbox.ResumeSandbox( ctx, @@ -46,6 +48,7 @@ func (f *ResumeSandbox) Sandbox( layerExecutor.devicePool, false, nil, + eventStore, ) if err != nil { return nil, fmt.Errorf("resume sandbox: %w", err) diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index d030ae60a1..0089053528 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -54,8 +54,9 @@ type Closeable interface { } const ( - defaultPort = 5008 - defaultProxyPort = 5007 + defaultPort = 5008 + defaultProxyPort = 5007 + defaultEventProxyPort = 5010 version = "0.1.0" @@ -70,6 +71,7 @@ var ( func main() { port := flag.Uint("port", defaultPort, "orchestrator server port") proxyPort := flag.Uint("proxy-port", defaultProxyPort, "orchestrator proxy port") + eventProxyPort := flag.Uint("event-proxy-port", defaultEventProxyPort, "orchestrator sandbox event proxy port") flag.Parse() if *port > math.MaxUint16 { @@ -80,7 +82,7 @@ func main() { log.Fatalf("%d is larger than maximum possible proxy port %d", proxyPort, math.MaxInt16) } - success := run(*port, *proxyPort) + success := run(*port, *proxyPort, *eventProxyPort) log.Println("Stopping orchestrator, success:", success) @@ -89,7 +91,7 @@ func main() { } } -func run(port, proxyPort uint) (success bool) { +func run(port, proxyPort, eventProxyPort uint) (success bool) { success = true services := service.GetServices() @@ -344,6 +346,11 @@ func run(port, proxyPort uint) (success bool) { } sbxEventsService := events.NewSandboxEventsService(featureFlags, redisPubSub, sandboxEventBatcher, globalLogger) + + sbxEventStore := events.NewSandboxEventStore(ctx, tracer, redisClient) + sbxEventHandlers := events.NewSandboxEventHandlers(ctx, sbxEventStore) + sbxEventProxy := events.NewSandboxEventProxy(eventProxyPort, sbxEventHandlers) + sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes) if err != nil { zap.L().Fatal("failed to create sandbox observer", zap.Error(err)) @@ -364,6 +371,7 @@ func run(port, proxyPort uint) (success bool) { Persistence: persistence, FeatureFlags: featureFlags, SbxEventsService: sbxEventsService, + SbxEventsStore: sbxEventStore, }, ) if err != nil { @@ -397,6 +405,8 @@ func run(port, proxyPort uint) (success bool) { sandboxObserver, limiter, sandboxEventBatcher, + sbxEventStore, + sbxEventProxy, ) // Initialize the template manager only if the service is enabled From b653e4b69a2b8d9c81207eedfdbc7597267a3b8c Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 15:25:37 -0700 Subject: [PATCH 05/27] pass eventstore through new Sandbox interfaces --- packages/orchestrator/internal/sandbox/sandbox.go | 12 +++++++----- .../internal/template/build/layer/create_sandbox.go | 2 -- .../internal/template/build/layer/interfaces.go | 2 -- .../internal/template/build/layer/layer_executor.go | 2 +- .../internal/template/build/layer/resume_sandbox.go | 12 ++++++------ .../internal/template/build/phases/steps/builder.go | 2 +- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index f9c3259cd2..a339c73c71 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -522,11 +522,13 @@ func ResumeSandbox( return nil, fmt.Errorf("failed to wait for sandbox start: %w", err) } - sandboxIP := ips.slot.HostIPString() - eventStore.SetSandboxIP(runtime.SandboxID, sandboxIP) - cleanup.AddPriority(func(ctx context.Context) error { - return eventStore.DelSandboxIP(sandboxIP) - }) + if eventStore != nil { + sandboxIP := ips.slot.HostIPString() + eventStore.SetSandboxIP(runtime.SandboxID, sandboxIP) + cleanup.AddPriority(func(ctx context.Context) error { + return eventStore.DelSandboxIP(sandboxIP) + }) + } go sbx.Checks.Start() diff --git a/packages/orchestrator/internal/template/build/layer/create_sandbox.go b/packages/orchestrator/internal/template/build/layer/create_sandbox.go index 5efb060196..46ad314e32 100644 --- a/packages/orchestrator/internal/template/build/layer/create_sandbox.go +++ b/packages/orchestrator/internal/template/build/layer/create_sandbox.go @@ -7,7 +7,6 @@ import ( "github.com/google/uuid" - "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/fc" @@ -41,7 +40,6 @@ func (cs *CreateSandbox) Sandbox( ctx context.Context, layerExecutor *LayerExecutor, sourceTemplate sbxtemplate.Template, - eventStore events.SandboxEventStore, ) (*sandbox.Sandbox, error) { // Create new memfile with the size of the sandbox RAM, this updates the underlying memfile. // This is ok as the sandbox is started from the beginning. diff --git a/packages/orchestrator/internal/template/build/layer/interfaces.go b/packages/orchestrator/internal/template/build/layer/interfaces.go index 972236200f..1cdb1ef5e0 100644 --- a/packages/orchestrator/internal/template/build/layer/interfaces.go +++ b/packages/orchestrator/internal/template/build/layer/interfaces.go @@ -4,7 +4,6 @@ import ( "context" "time" - "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/template" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/metadata" @@ -20,7 +19,6 @@ type SandboxCreator interface { ctx context.Context, layerExecutor *LayerExecutor, template sbxtemplate.Template, - eventStore events.SandboxEventStore, ) (*sandbox.Sandbox, error) } diff --git a/packages/orchestrator/internal/template/build/layer/layer_executor.go b/packages/orchestrator/internal/template/build/layer/layer_executor.go index d467b16561..312b792eb7 100644 --- a/packages/orchestrator/internal/template/build/layer/layer_executor.go +++ b/packages/orchestrator/internal/template/build/layer/layer_executor.go @@ -81,7 +81,7 @@ func (lb *LayerExecutor) BuildLayer( } // Create or resume sandbox - sbx, err := cmd.SandboxCreator.Sandbox(ctx, lb, localTemplate, lb.sbxEventsStore) + sbx, err := cmd.SandboxCreator.Sandbox(ctx, lb, localTemplate) if err != nil { return metadata.Template{}, err } diff --git a/packages/orchestrator/internal/template/build/layer/resume_sandbox.go b/packages/orchestrator/internal/template/build/layer/resume_sandbox.go index ef92094916..4dca8799b4 100644 --- a/packages/orchestrator/internal/template/build/layer/resume_sandbox.go +++ b/packages/orchestrator/internal/template/build/layer/resume_sandbox.go @@ -16,21 +16,21 @@ import ( // ResumeSandbox creates sandboxes for resuming existing templates type ResumeSandbox struct { - config sandbox.Config - timeout time.Duration + config sandbox.Config + timeout time.Duration + eventStore events.SandboxEventStore } var _ SandboxCreator = (*ResumeSandbox)(nil) -func NewResumeSandbox(config sandbox.Config, timeout time.Duration) *ResumeSandbox { - return &ResumeSandbox{config: config, timeout: timeout} +func NewResumeSandbox(config sandbox.Config, timeout time.Duration, eventStore events.SandboxEventStore) *ResumeSandbox { + return &ResumeSandbox{config: config, timeout: timeout, eventStore: eventStore} } func (rs *ResumeSandbox) Sandbox( ctx context.Context, layerExecutor *LayerExecutor, template sbxtemplate.Template, - eventStore events.SandboxEventStore, ) (*sandbox.Sandbox, error) { sbx, err := sandbox.ResumeSandbox( ctx, @@ -49,7 +49,7 @@ func (rs *ResumeSandbox) Sandbox( layerExecutor.devicePool, false, nil, - eventStore, + rs.eventStore, ) if err != nil { return nil, fmt.Errorf("resume sandbox: %w", err) diff --git a/packages/orchestrator/internal/template/build/phases/steps/builder.go b/packages/orchestrator/internal/template/build/phases/steps/builder.go index fe0a3494e9..789da9b201 100644 --- a/packages/orchestrator/internal/template/build/phases/steps/builder.go +++ b/packages/orchestrator/internal/template/build/phases/steps/builder.go @@ -177,7 +177,7 @@ func (sb *StepBuilder) Build( }, ) } else { - sandboxCreator = layer.NewResumeSandbox(sbxConfig, layerTimeout) + sandboxCreator = layer.NewResumeSandbox(sbxConfig, layerTimeout, nil) } actionExecutor := layer.NewFunctionAction(func(ctx context.Context, sbx *sandbox.Sandbox, meta metadata.Template) (metadata.Template, error) { From d73aea7f8199113019524fd47d1dbbf74064dd0d Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 16:52:23 -0700 Subject: [PATCH 06/27] address context changes --- .../events/sandbox_events_handlers.go | 9 ++-- .../internal/events/sandbox_events_store.go | 54 +++++++++---------- .../orchestrator/internal/sandbox/sandbox.go | 14 ++++- packages/orchestrator/main.go | 4 +- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/packages/orchestrator/internal/events/sandbox_events_handlers.go b/packages/orchestrator/internal/events/sandbox_events_handlers.go index 86694faa53..e6b56521f7 100644 --- a/packages/orchestrator/internal/events/sandbox_events_handlers.go +++ b/packages/orchestrator/internal/events/sandbox_events_handlers.go @@ -1,7 +1,6 @@ package events import ( - "context" "encoding/json" "io" "net/http" @@ -27,7 +26,7 @@ func (h *DefaultSandboxEventHandler) Path() string { func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { addr := r.RemoteAddr ip := strings.Split(addr, ":")[0] - sandboxID, err := h.store.GetSandboxIP(ip) + sandboxID, err := h.store.GetSandboxIP(r.Context(), ip) if err != nil { zap.L().Error("Failed to get sandbox ID from IP", zap.Error(err)) http.Error(w, "Error handling event", http.StatusInternalServerError) @@ -37,7 +36,7 @@ func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http. zap.L().Debug("Received request from sandbox", zap.String("sandbox_id", sandboxID), zap.String("ip", ip)) if r.Method == http.MethodGet { - events, err := h.store.GetLastNEvents(sandboxID, 10) + events, err := h.store.GetLastNEvents(r.Context(), sandboxID, 10) if err != nil { zap.L().Error("Failed to get event data for sandbox "+sandboxID, zap.Error(err)) http.Error(w, "Failed to get event data for sandbox "+sandboxID, http.StatusInternalServerError) @@ -84,7 +83,7 @@ func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http. } // Store in Redis with sandboxID as key - err = h.store.AddEvent(sandboxID, &eventData, 0) + err = h.store.AddEvent(r.Context(), sandboxID, &eventData, 0) if err != nil { zap.L().Error("Failed to store event data", zap.Error(err)) http.Error(w, "Failed to store event data", http.StatusInternalServerError) @@ -95,7 +94,7 @@ func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http. w.Write([]byte(`{"event_ack":true}`)) } -func NewSandboxEventHandlers(ctx context.Context, store SandboxEventStore) []SandboxEventHandler { +func NewSandboxEventHandlers(store SandboxEventStore) []SandboxEventHandler { return []SandboxEventHandler{ &DefaultSandboxEventHandler{store}, } diff --git a/packages/orchestrator/internal/events/sandbox_events_store.go b/packages/orchestrator/internal/events/sandbox_events_store.go index 4523da3a68..f2fd4e6154 100644 --- a/packages/orchestrator/internal/events/sandbox_events_store.go +++ b/packages/orchestrator/internal/events/sandbox_events_store.go @@ -26,49 +26,47 @@ func (i SandboxEvent) MarshalBinary() ([]byte, error) { } type sandboxEventStore struct { - ctx context.Context tracer trace.Tracer redisClient redis.UniversalClient } type SandboxEventStore interface { - SetSandboxIP(sandboxId string, ip string) error - GetSandboxIP(sandboxId string) (string, error) - DelSandboxIP(sandboxId string) error + SetSandboxIP(ctx context.Context, sandboxId string, ip string) error + GetSandboxIP(ctx context.Context, sandboxId string) (string, error) + DelSandboxIP(ctx context.Context, sandboxId string) error - GetLastEvent(sandboxId string) (*SandboxEvent, error) - GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) - AddEvent(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error - DelEvent(sandboxId string) error + GetLastEvent(ctx context.Context, sandboxId string) (*SandboxEvent, error) + GetLastNEvents(ctx context.Context, sandboxId string, n int) ([]*SandboxEvent, error) + AddEvent(ctx context.Context, sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error + DelEvent(ctx context.Context, sandboxId string) error Close(ctx context.Context) error } -func NewSandboxEventStore(ctx context.Context, tracer trace.Tracer, redisClient redis.UniversalClient) SandboxEventStore { +func NewSandboxEventStore(tracer trace.Tracer, redisClient redis.UniversalClient) SandboxEventStore { return &sandboxEventStore{ - ctx: ctx, tracer: tracer, redisClient: redisClient, } } -func (c *sandboxEventStore) SetSandboxIP(sandboxId string, ip string) error { - return c.redisClient.Set(c.ctx, IPPrefix+ip, sandboxId, cacheTL).Err() +func (c *sandboxEventStore) SetSandboxIP(ctx context.Context, sandboxId string, ip string) error { + return c.redisClient.Set(ctx, IPPrefix+ip, sandboxId, cacheTL).Err() } -func (c *sandboxEventStore) GetSandboxIP(ip string) (string, error) { - return c.redisClient.Get(c.ctx, IPPrefix+ip).Result() +func (c *sandboxEventStore) GetSandboxIP(ctx context.Context, ip string) (string, error) { + return c.redisClient.Get(ctx, IPPrefix+ip).Result() } -func (c *sandboxEventStore) DelSandboxIP(ip string) error { - return c.redisClient.Del(c.ctx, IPPrefix+ip).Err() +func (c *sandboxEventStore) DelSandboxIP(ctx context.Context, ip string) error { + return c.redisClient.Del(ctx, IPPrefix+ip).Err() } -func (c *sandboxEventStore) GetLastEvent(sandboxId string) (*SandboxEvent, error) { - _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last") +func (c *sandboxEventStore) GetLastEvent(ctx context.Context, sandboxId string) (*SandboxEvent, error) { + _, span := c.tracer.Start(ctx, "sandbox-event-get-last") defer span.End() - result, err := c.redisClient.ZRevRangeWithScores(c.ctx, EventPrefix+sandboxId, 0, 0).Result() + result, err := c.redisClient.ZRevRangeWithScores(ctx, EventPrefix+sandboxId, 0, 0).Result() if err != nil { return nil, err } @@ -85,11 +83,11 @@ func (c *sandboxEventStore) GetLastEvent(sandboxId string) (*SandboxEvent, error return &event, nil } -func (c *sandboxEventStore) GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) { - _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last-n") +func (c *sandboxEventStore) GetLastNEvents(ctx context.Context, sandboxId string, n int) ([]*SandboxEvent, error) { + _, span := c.tracer.Start(ctx, "sandbox-event-get-last-n") defer span.End() - result, err := c.redisClient.ZRevRangeWithScores(c.ctx, EventPrefix+sandboxId, 0, int64(n-1)).Result() + result, err := c.redisClient.ZRevRangeWithScores(ctx, EventPrefix+sandboxId, 0, int64(n-1)).Result() if err != nil { return nil, err } @@ -111,21 +109,21 @@ func (c *sandboxEventStore) GetLastNEvents(sandboxId string, n int) ([]*SandboxE return events, nil } -func (c *sandboxEventStore) AddEvent(sandboxId string, event *SandboxEvent, expiration time.Duration) error { - _, span := c.tracer.Start(c.ctx, "sandbox-event-store") +func (c *sandboxEventStore) AddEvent(ctx context.Context, sandboxId string, event *SandboxEvent, expiration time.Duration) error { + _, span := c.tracer.Start(ctx, "sandbox-event-store") defer span.End() - return c.redisClient.ZAdd(c.ctx, EventPrefix+sandboxId, redis.Z{ + return c.redisClient.ZAdd(ctx, EventPrefix+sandboxId, redis.Z{ Score: float64(time.Now().UnixNano()), Member: event, }).Err() } -func (c *sandboxEventStore) DelEvent(sandboxId string) error { - _, span := c.tracer.Start(c.ctx, "sandbox-event-delete") +func (c *sandboxEventStore) DelEvent(ctx context.Context, sandboxId string) error { + _, span := c.tracer.Start(ctx, "sandbox-event-delete") defer span.End() - return c.redisClient.Del(c.ctx, EventPrefix+sandboxId).Err() + return c.redisClient.Del(ctx, EventPrefix+sandboxId).Err() } func (c *sandboxEventStore) Close(ctx context.Context) error { diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index a339c73c71..e3bb2f7f21 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -524,9 +524,19 @@ func ResumeSandbox( if eventStore != nil { sandboxIP := ips.slot.HostIPString() - eventStore.SetSandboxIP(runtime.SandboxID, sandboxIP) + + ctx = context.WithoutCancel(ctx) + err = eventStore.SetSandboxIP(context.WithoutCancel(ctx), runtime.SandboxID, sandboxIP) + if err != nil { + // soft fail to not block this critical path + zap.L().Error("failed to set sandbox IP", zap.String("sandbox_id", runtime.SandboxID), zap.String("ip", ips.slot.HostIPString()), zap.Error(err)) + } cleanup.AddPriority(func(ctx context.Context) error { - return eventStore.DelSandboxIP(sandboxIP) + err = eventStore.DelSandboxIP(ctx, sandboxIP) + if err != nil { + return fmt.Errorf("failed to delete sandbox IP: %w", err) + } + return nil }) } diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 9a383406fd..334d536891 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -347,8 +347,8 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { sbxEventsService := events.NewSandboxEventsService(featureFlags, redisPubSub, sandboxEventBatcher, globalLogger) - sbxEventStore := events.NewSandboxEventStore(ctx, tracer, redisClient) - sbxEventHandlers := events.NewSandboxEventHandlers(ctx, sbxEventStore) + sbxEventStore := events.NewSandboxEventStore(tracer, redisClient) + sbxEventHandlers := events.NewSandboxEventHandlers(sbxEventStore) sbxEventProxy := events.NewSandboxEventProxy(eventProxyPort, sbxEventHandlers) sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes) From b7d9dc8124383ceb73316303888865162eb9bc90 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 17:18:17 -0700 Subject: [PATCH 07/27] add consts for event proxy port --- packages/orchestrator/internal/consts.go | 5 +++++ .../orchestrator/internal/sandbox/network/network.go | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go index 783d824e9f..a0c3f72414 100644 --- a/packages/orchestrator/internal/consts.go +++ b/packages/orchestrator/internal/consts.go @@ -8,8 +8,13 @@ const ( // Private IP don't leave the sandbox through the network bridge, so we use a reserved IP address for it: // See TEST-NET-3 on https://en.wikipedia.org/wiki/Reserved_IP_addresses defaultSandboxEventIP = "203.0.113.0" + defaultEventProxyPort = "5010" ) func GetSandboxEventIP() string { return env.GetEnv("SANDBOX_EVENT_IP", defaultSandboxEventIP) } + +func GetEventProxyPort() string { + return env.GetEnv("SANDBOX_EVENT_PROXY_PORT", defaultEventProxyPort) +} diff --git a/packages/orchestrator/internal/sandbox/network/network.go b/packages/orchestrator/internal/sandbox/network/network.go index ec9d5459d8..7376b004e3 100644 --- a/packages/orchestrator/internal/sandbox/network/network.go +++ b/packages/orchestrator/internal/sandbox/network/network.go @@ -12,6 +12,7 @@ import ( "go.uber.org/zap" "github.com/e2b-dev/infra/packages/orchestrator/internal" + consts "github.com/e2b-dev/infra/packages/orchestrator/internal" ) func (s *Slot) CreateNetwork() error { @@ -216,11 +217,14 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } - // Redirect traffic destined for event server + // Redirect traffic destined for event proxy server eventIP := internal.GetSandboxEventIP() - err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") + err = tables.Append( + "nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", + "-j", "REDIRECT", "--to-port", consts.GetEventProxyPort(), + ) if err != nil { - return fmt.Errorf("error creating HTTP redirect rule to sandbox event server: %w", err) + return fmt.Errorf("error creating HTTP redirect rule to sandbox event proxy server: %w", err) } return nil From b33479deb898f9157c4825672f101d4b4a2253d2 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 17:27:15 -0700 Subject: [PATCH 08/27] fix consts import --- packages/orchestrator/internal/sandbox/network/network.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/network/network.go b/packages/orchestrator/internal/sandbox/network/network.go index 7376b004e3..29e2fc425b 100644 --- a/packages/orchestrator/internal/sandbox/network/network.go +++ b/packages/orchestrator/internal/sandbox/network/network.go @@ -11,7 +11,6 @@ import ( "github.com/vishvananda/netns" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/orchestrator/internal" consts "github.com/e2b-dev/infra/packages/orchestrator/internal" ) @@ -218,7 +217,7 @@ func (s *Slot) CreateNetwork() error { } // Redirect traffic destined for event proxy server - eventIP := internal.GetSandboxEventIP() + eventIP := consts.GetSandboxEventIP() err = tables.Append( "nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", "-j", "REDIRECT", "--to-port", consts.GetEventProxyPort(), From 46e20cd5248830935884b8232569de1439626c1c Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 18:18:05 -0700 Subject: [PATCH 09/27] refactor event proxy instantiation, naming and default handler --- .../internal/events/events_service.go | 2 +- .../internal/events/mock_events_service.go | 12 +++---- .../events/sandbox_events_handlers.go | 10 +++--- .../internal/events/sandbox_events_proxy.go | 3 +- packages/orchestrator/internal/server/main.go | 34 +++++++++--------- .../orchestrator/internal/server/sandboxes.go | 10 +++--- packages/orchestrator/main.go | 36 +++++++++---------- 7 files changed, 52 insertions(+), 55 deletions(-) diff --git a/packages/orchestrator/internal/events/events_service.go b/packages/orchestrator/internal/events/events_service.go index 715340d582..e197fdc4cf 100644 --- a/packages/orchestrator/internal/events/events_service.go +++ b/packages/orchestrator/internal/events/events_service.go @@ -4,7 +4,7 @@ import ( "context" ) -type EventsService[T any] interface { +type EventService[T any] interface { HandleEvent(ctx context.Context, event T) Close(ctx context.Context) error } diff --git a/packages/orchestrator/internal/events/mock_events_service.go b/packages/orchestrator/internal/events/mock_events_service.go index 7ce807d90e..11a8d947df 100644 --- a/packages/orchestrator/internal/events/mock_events_service.go +++ b/packages/orchestrator/internal/events/mock_events_service.go @@ -4,23 +4,23 @@ import ( "context" ) -// MockEventsService implements EventsService interface for testing -type MockEventsService[T any] struct { +// MockEventService implements EventsService interface for testing +type MockEventService[T any] struct { HandleEventFunc func(ctx context.Context, event T) error CloseFunc func(ctx context.Context) error } -func NewMockEventsService[T any]() *MockEventsService[T] { - return &MockEventsService[T]{ +func NewMockEventsService[T any]() *MockEventService[T] { + return &MockEventService[T]{ HandleEventFunc: func(ctx context.Context, event T) error { return nil }, CloseFunc: func(ctx context.Context) error { return nil }, } } -func (m *MockEventsService[T]) HandleEvent(ctx context.Context, event T) error { +func (m *MockEventService[T]) HandleEvent(ctx context.Context, event T) error { return m.HandleEventFunc(ctx, event) } -func (m *MockEventsService[T]) Close(ctx context.Context) error { +func (m *MockEventService[T]) Close(ctx context.Context) error { return m.CloseFunc(ctx) } diff --git a/packages/orchestrator/internal/events/sandbox_events_handlers.go b/packages/orchestrator/internal/events/sandbox_events_handlers.go index e6b56521f7..b9568c4805 100644 --- a/packages/orchestrator/internal/events/sandbox_events_handlers.go +++ b/packages/orchestrator/internal/events/sandbox_events_handlers.go @@ -19,6 +19,10 @@ type DefaultSandboxEventHandler struct { store SandboxEventStore } +func NewDefaultSandboxEventHandler(store SandboxEventStore) *DefaultSandboxEventHandler { + return &DefaultSandboxEventHandler{store} +} + func (h *DefaultSandboxEventHandler) Path() string { return "/" } @@ -93,9 +97,3 @@ func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http. w.WriteHeader(http.StatusCreated) w.Write([]byte(`{"event_ack":true}`)) } - -func NewSandboxEventHandlers(store SandboxEventStore) []SandboxEventHandler { - return []SandboxEventHandler{ - &DefaultSandboxEventHandler{store}, - } -} diff --git a/packages/orchestrator/internal/events/sandbox_events_proxy.go b/packages/orchestrator/internal/events/sandbox_events_proxy.go index 8e0a7ba953..694d0c3ff9 100644 --- a/packages/orchestrator/internal/events/sandbox_events_proxy.go +++ b/packages/orchestrator/internal/events/sandbox_events_proxy.go @@ -11,9 +11,10 @@ type SandboxEventProxy struct { server *http.Server } -func NewSandboxEventProxy(port uint, handlers []SandboxEventHandler) *SandboxEventProxy { +func NewSandboxEventProxy(port uint, store SandboxEventStore, handlers ...SandboxEventHandler) *SandboxEventProxy { mux := http.NewServeMux() + handlers = append(handlers, NewDefaultSandboxEventHandler(store)) for _, handler := range handlers { mux.HandleFunc(handler.Path(), handler.HandlerFunc) } diff --git a/packages/orchestrator/internal/server/main.go b/packages/orchestrator/internal/server/main.go index 5709d229ab..55ebfd3db8 100644 --- a/packages/orchestrator/internal/server/main.go +++ b/packages/orchestrator/internal/server/main.go @@ -38,9 +38,9 @@ type server struct { devicePool *nbd.DevicePool persistence storage.StorageProvider featureFlags *featureflags.Client - sbxEventsService events.EventsService[event.SandboxEvent] + sbxEventService events.EventService[event.SandboxEvent] startingSandboxes *semaphore.Weighted - sbxEventsStore events.SandboxEventStore + sbxEventStore events.SandboxEventStore } type Service struct { @@ -57,19 +57,19 @@ type Service struct { } type ServiceConfig struct { - GRPC *grpcserver.GRPCServer - Tel *telemetry.Client - NetworkPool *network.Pool - DevicePool *nbd.DevicePool - TemplateCache *template.Cache - Tracer trace.Tracer - Info *service.ServiceInfo - Proxy *proxy.SandboxProxy - Sandboxes *smap.Map[*sandbox.Sandbox] - Persistence storage.StorageProvider - FeatureFlags *featureflags.Client - SbxEventsService events.EventsService[event.SandboxEvent] - SbxEventsStore events.SandboxEventStore + GRPC *grpcserver.GRPCServer + Tel *telemetry.Client + NetworkPool *network.Pool + DevicePool *nbd.DevicePool + TemplateCache *template.Cache + Tracer trace.Tracer + Info *service.ServiceInfo + Proxy *proxy.SandboxProxy + Sandboxes *smap.Map[*sandbox.Sandbox] + Persistence storage.StorageProvider + FeatureFlags *featureflags.Client + SbxEventService events.EventService[event.SandboxEvent] + SbxEventStore events.SandboxEventStore } func New( @@ -91,9 +91,9 @@ func New( devicePool: cfg.DevicePool, persistence: cfg.Persistence, featureFlags: cfg.FeatureFlags, - sbxEventsService: cfg.SbxEventsService, + sbxEventService: cfg.SbxEventService, + sbxEventStore: cfg.SbxEventStore, startingSandboxes: semaphore.NewWeighted(maxStartingInstancesPerNode), - sbxEventsStore: cfg.SbxEventsStore, } meter := cfg.Tel.MeterProvider.Meter("orchestrator.sandbox") diff --git a/packages/orchestrator/internal/server/sandboxes.go b/packages/orchestrator/internal/server/sandboxes.go index 47e1902a0b..0a87b011cf 100644 --- a/packages/orchestrator/internal/server/sandboxes.go +++ b/packages/orchestrator/internal/server/sandboxes.go @@ -121,7 +121,7 @@ func (s *server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ s.devicePool, metricsWriteFlag, req.Sandbox, - s.sbxEventsStore, + s.sbxEventStore, ) if err != nil { err := errors.Join(err, context.Cause(ctx)) @@ -180,7 +180,7 @@ func (s *server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ buildId = sbx.APIStoredConfig.BuildId } - go s.sbxEventsService.HandleEvent(ctx, event.SandboxEvent{ + go s.sbxEventService.HandleEvent(ctx, event.SandboxEvent{ Timestamp: time.Now().UTC(), SandboxID: sbx.Runtime.SandboxID, SandboxExecutionID: sbx.Runtime.ExecutionID, @@ -228,7 +228,7 @@ func (s *server) Update(ctx context.Context, req *orchestrator.SandboxUpdateRequ buildId = sbx.APIStoredConfig.BuildId } - go s.sbxEventsService.HandleEvent(ctx, event.SandboxEvent{ + go s.sbxEventService.HandleEvent(ctx, event.SandboxEvent{ Timestamp: time.Now().UTC(), SandboxID: sbx.Runtime.SandboxID, SandboxExecutionID: sbx.Runtime.ExecutionID, @@ -323,7 +323,7 @@ func (s *server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR eventCtx := context.WithoutCancel(ctx) - go s.sbxEventsService.HandleEvent(eventCtx, event.SandboxEvent{ + go s.sbxEventService.HandleEvent(eventCtx, event.SandboxEvent{ Timestamp: time.Now().UTC(), SandboxID: sbx.Runtime.SandboxID, SandboxExecutionID: sbx.Runtime.ExecutionID, @@ -438,7 +438,7 @@ func (s *server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest } eventCtx := context.WithoutCancel(ctx) - go s.sbxEventsService.HandleEvent(eventCtx, event.SandboxEvent{ + go s.sbxEventService.HandleEvent(eventCtx, event.SandboxEvent{ Timestamp: time.Now().UTC(), SandboxID: sbx.Runtime.SandboxID, SandboxExecutionID: sbx.Runtime.ExecutionID, diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 334d536891..8b7b14e38c 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -345,11 +345,9 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { redisPubSub = pubsub.NewMockPubSub[event.SandboxEvent, webhooks.SandboxWebhooksMetaData]() } - sbxEventsService := events.NewSandboxEventsService(featureFlags, redisPubSub, sandboxEventBatcher, globalLogger) - - sbxEventStore := events.NewSandboxEventStore(tracer, redisClient) - sbxEventHandlers := events.NewSandboxEventHandlers(sbxEventStore) - sbxEventProxy := events.NewSandboxEventProxy(eventProxyPort, sbxEventHandlers) + sbxEventService := events.NewSandboxEventsService(featureFlags, redisPubSub, sandboxEventBatcher, globalLogger) + sbxEventtore := events.NewSandboxEventStore(tracer, redisClient) + sbxEventProxy := events.NewSandboxEventProxy(eventProxyPort, sbxEventtore) sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes) if err != nil { @@ -359,19 +357,19 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { _, err = server.New( ctx, server.ServiceConfig{ - GRPC: grpcSrv, - Tel: tel, - NetworkPool: networkPool, - DevicePool: devicePool, - TemplateCache: templateCache, - Tracer: tracer, - Info: serviceInfo, - Proxy: sandboxProxy, - Sandboxes: sandboxes, - Persistence: persistence, - FeatureFlags: featureFlags, - SbxEventsService: sbxEventsService, - SbxEventsStore: sbxEventStore, + GRPC: grpcSrv, + Tel: tel, + NetworkPool: networkPool, + DevicePool: devicePool, + TemplateCache: templateCache, + Tracer: tracer, + Info: serviceInfo, + Proxy: sandboxProxy, + Sandboxes: sandboxes, + Persistence: persistence, + FeatureFlags: featureFlags, + SbxEventService: sbxEventService, + SbxEventStore: sbxEventtore, }, ) if err != nil { @@ -405,7 +403,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { sandboxObserver, limiter, sandboxEventBatcher, - sbxEventStore, + sbxEventtore, sbxEventProxy, ) From 2391dde0e94c66f40e8815f285e3e5770293c53b Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 19:15:00 -0700 Subject: [PATCH 10/27] start event proxy server in main --- .../internal/sandbox/network/network.go | 4 ++-- packages/orchestrator/main.go | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/network/network.go b/packages/orchestrator/internal/sandbox/network/network.go index 29e2fc425b..6b0eba4f29 100644 --- a/packages/orchestrator/internal/sandbox/network/network.go +++ b/packages/orchestrator/internal/sandbox/network/network.go @@ -217,9 +217,9 @@ func (s *Slot) CreateNetwork() error { } // Redirect traffic destined for event proxy server - eventIP := consts.GetSandboxEventIP() err = tables.Append( - "nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", + "nat", "PREROUTING", "-i", s.VethName(), + "-p", "tcp", "-d", consts.GetSandboxEventIP(), "--dport", "80", "-j", "REDIRECT", "--to-port", consts.GetEventProxyPort(), ) if err != nil { diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 8b7b14e38c..0ae4fe8c5c 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -475,6 +475,24 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { return nil }) + g.Go(func() error { + sbxEventProxyErr := sbxEventProxy.Start() + if sbxEventProxyErr != nil && !errors.Is(sbxEventProxyErr, http.ErrServerClosed) { + zap.L().Error("sandbox event proxy error", zap.Error(sbxEventProxyErr)) + + select { + case serviceError <- sbxEventProxyErr: + default: + // Don't block if the serviceError channel is already closed + // or if the error is already sent + } + + return sbxEventProxyErr + } + + return nil + }) + // Wait for the shutdown signal or if some service fails select { case <-sig.Done(): From 08da808822d6cf8ccaf9f9afb21cce2e6ba9fba9 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 3 Sep 2025 19:31:06 -0700 Subject: [PATCH 11/27] rename event store args for readability --- .../events/sandbox_events_handlers.go | 6 +-- .../internal/events/sandbox_events_store.go | 42 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/orchestrator/internal/events/sandbox_events_handlers.go b/packages/orchestrator/internal/events/sandbox_events_handlers.go index b9568c4805..fdbe51fa0e 100644 --- a/packages/orchestrator/internal/events/sandbox_events_handlers.go +++ b/packages/orchestrator/internal/events/sandbox_events_handlers.go @@ -29,15 +29,15 @@ func (h *DefaultSandboxEventHandler) Path() string { func (h *DefaultSandboxEventHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { addr := r.RemoteAddr - ip := strings.Split(addr, ":")[0] - sandboxID, err := h.store.GetSandboxIP(r.Context(), ip) + sandboxIP := strings.Split(addr, ":")[0] + sandboxID, err := h.store.GetSandboxID(r.Context(), sandboxIP) if err != nil { zap.L().Error("Failed to get sandbox ID from IP", zap.Error(err)) http.Error(w, "Error handling event", http.StatusInternalServerError) return } - zap.L().Debug("Received request from sandbox", zap.String("sandbox_id", sandboxID), zap.String("ip", ip)) + zap.L().Debug("Received request from sandbox", zap.String("sandbox_id", sandboxID), zap.String("sandbox_ip", sandboxIP)) if r.Method == http.MethodGet { events, err := h.store.GetLastNEvents(r.Context(), sandboxID, 10) diff --git a/packages/orchestrator/internal/events/sandbox_events_store.go b/packages/orchestrator/internal/events/sandbox_events_store.go index f2fd4e6154..2751924f69 100644 --- a/packages/orchestrator/internal/events/sandbox_events_store.go +++ b/packages/orchestrator/internal/events/sandbox_events_store.go @@ -31,14 +31,14 @@ type sandboxEventStore struct { } type SandboxEventStore interface { - SetSandboxIP(ctx context.Context, sandboxId string, ip string) error - GetSandboxIP(ctx context.Context, sandboxId string) (string, error) - DelSandboxIP(ctx context.Context, sandboxId string) error + SetSandboxIP(ctx context.Context, sandboxID string, sandboxIP string) error + GetSandboxID(ctx context.Context, sandboxIP string) (string, error) + DelSandboxIP(ctx context.Context, sandboxIP string) error - GetLastEvent(ctx context.Context, sandboxId string) (*SandboxEvent, error) - GetLastNEvents(ctx context.Context, sandboxId string, n int) ([]*SandboxEvent, error) - AddEvent(ctx context.Context, sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error - DelEvent(ctx context.Context, sandboxId string) error + GetLastEvent(ctx context.Context, sandboxID string) (*SandboxEvent, error) + GetLastNEvents(ctx context.Context, sandboxID string, n int) ([]*SandboxEvent, error) + AddEvent(ctx context.Context, sandboxID string, SandboxEvent *SandboxEvent, expiration time.Duration) error + DelEvent(ctx context.Context, sandboxID string) error Close(ctx context.Context) error } @@ -50,23 +50,23 @@ func NewSandboxEventStore(tracer trace.Tracer, redisClient redis.UniversalClient } } -func (c *sandboxEventStore) SetSandboxIP(ctx context.Context, sandboxId string, ip string) error { - return c.redisClient.Set(ctx, IPPrefix+ip, sandboxId, cacheTL).Err() +func (c *sandboxEventStore) SetSandboxIP(ctx context.Context, sandboxID string, sandboxIP string) error { + return c.redisClient.Set(ctx, IPPrefix+sandboxIP, sandboxID, cacheTL).Err() } -func (c *sandboxEventStore) GetSandboxIP(ctx context.Context, ip string) (string, error) { - return c.redisClient.Get(ctx, IPPrefix+ip).Result() +func (c *sandboxEventStore) GetSandboxID(ctx context.Context, sandboxIP string) (string, error) { + return c.redisClient.Get(ctx, IPPrefix+sandboxIP).Result() } -func (c *sandboxEventStore) DelSandboxIP(ctx context.Context, ip string) error { - return c.redisClient.Del(ctx, IPPrefix+ip).Err() +func (c *sandboxEventStore) DelSandboxIP(ctx context.Context, sandboxIP string) error { + return c.redisClient.Del(ctx, IPPrefix+sandboxIP).Err() } -func (c *sandboxEventStore) GetLastEvent(ctx context.Context, sandboxId string) (*SandboxEvent, error) { +func (c *sandboxEventStore) GetLastEvent(ctx context.Context, sandboxID string) (*SandboxEvent, error) { _, span := c.tracer.Start(ctx, "sandbox-event-get-last") defer span.End() - result, err := c.redisClient.ZRevRangeWithScores(ctx, EventPrefix+sandboxId, 0, 0).Result() + result, err := c.redisClient.ZRevRangeWithScores(ctx, EventPrefix+sandboxID, 0, 0).Result() if err != nil { return nil, err } @@ -83,11 +83,11 @@ func (c *sandboxEventStore) GetLastEvent(ctx context.Context, sandboxId string) return &event, nil } -func (c *sandboxEventStore) GetLastNEvents(ctx context.Context, sandboxId string, n int) ([]*SandboxEvent, error) { +func (c *sandboxEventStore) GetLastNEvents(ctx context.Context, sandboxID string, n int) ([]*SandboxEvent, error) { _, span := c.tracer.Start(ctx, "sandbox-event-get-last-n") defer span.End() - result, err := c.redisClient.ZRevRangeWithScores(ctx, EventPrefix+sandboxId, 0, int64(n-1)).Result() + result, err := c.redisClient.ZRevRangeWithScores(ctx, EventPrefix+sandboxID, 0, int64(n-1)).Result() if err != nil { return nil, err } @@ -109,21 +109,21 @@ func (c *sandboxEventStore) GetLastNEvents(ctx context.Context, sandboxId string return events, nil } -func (c *sandboxEventStore) AddEvent(ctx context.Context, sandboxId string, event *SandboxEvent, expiration time.Duration) error { +func (c *sandboxEventStore) AddEvent(ctx context.Context, sandboxID string, event *SandboxEvent, expiration time.Duration) error { _, span := c.tracer.Start(ctx, "sandbox-event-store") defer span.End() - return c.redisClient.ZAdd(ctx, EventPrefix+sandboxId, redis.Z{ + return c.redisClient.ZAdd(ctx, EventPrefix+sandboxID, redis.Z{ Score: float64(time.Now().UnixNano()), Member: event, }).Err() } -func (c *sandboxEventStore) DelEvent(ctx context.Context, sandboxId string) error { +func (c *sandboxEventStore) DelEvent(ctx context.Context, sandboxID string) error { _, span := c.tracer.Start(ctx, "sandbox-event-delete") defer span.End() - return c.redisClient.Del(ctx, EventPrefix+sandboxId).Err() + return c.redisClient.Del(ctx, EventPrefix+sandboxID).Err() } func (c *sandboxEventStore) Close(ctx context.Context) error { From 6748272b998cd08f101e8f274031e51a4268af43 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 8 Sep 2025 18:42:24 -0700 Subject: [PATCH 12/27] pass event address fc mmdds in orchestrator --- packages/orchestrator/internal/consts.go | 6 +++--- packages/orchestrator/internal/sandbox/fc/mmds.go | 1 + packages/orchestrator/internal/sandbox/sandbox.go | 2 ++ .../internal/template/build/core/rootfs/rootfs.go | 6 +----- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go index 783d824e9f..6cc227dfe5 100644 --- a/packages/orchestrator/internal/consts.go +++ b/packages/orchestrator/internal/consts.go @@ -5,9 +5,9 @@ import ( ) const ( - // Private IP don't leave the sandbox through the network bridge, so we use a reserved IP address for it: - // See TEST-NET-3 on https://en.wikipedia.org/wiki/Reserved_IP_addresses - defaultSandboxEventIP = "203.0.113.0" + // Same range used by metadata services in Firecracker/EC2 + // https://en.m.wikipedia.org/wiki/Link-local_address + defaultSandboxEventIP = "169.254.0.7" ) func GetSandboxEventIP() string { diff --git a/packages/orchestrator/internal/sandbox/fc/mmds.go b/packages/orchestrator/internal/sandbox/fc/mmds.go index 859dd477bf..e97135a99b 100644 --- a/packages/orchestrator/internal/sandbox/fc/mmds.go +++ b/packages/orchestrator/internal/sandbox/fc/mmds.go @@ -11,6 +11,7 @@ type MmdsMetadata struct { LogsCollectorAddress string `json:"address"` TraceId string `json:"traceID"` TeamId string `json:"teamID"` + EventsAddress string `json:"eventsAddress"` } func (mm MmdsMetadata) LoggerMetadata() sbxlogger.SandboxMetadata { diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index 71d7ce9cfd..9e707bfdd7 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -13,6 +13,7 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/orchestrator/internal" globalconfig "github.com/e2b-dev/infra/packages/orchestrator/internal/config" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" @@ -457,6 +458,7 @@ func ResumeSandbox( LogsCollectorAddress: fmt.Sprintf("http://%s", logsCollectorIP), TraceId: traceID, TeamId: runtime.TeamID, + EventsAddress: internal.GetSandboxEventIP(), }, fcUffdPath, snapfile, diff --git a/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go b/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go index 928a7de0d1..c25b2a25a4 100644 --- a/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go +++ b/packages/orchestrator/internal/template/build/core/rootfs/rootfs.go @@ -14,7 +14,6 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/orchestrator/internal" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/config" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/filesystem" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/core/oci" @@ -199,8 +198,6 @@ ExecStart=-/sbin/agetty --noissue --autologin root %I 115200,38400,9600 vt102 ` hostname := "e2b.local" - eventProxyHostname := "events.e2b.dev" - eventIP := internal.GetSandboxEventIP() hosts := fmt.Sprintf(`127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -209,8 +206,7 @@ ff00:: ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters 127.0.1.1 %s -%s %s -`, hostname, eventIP, eventProxyHostname) +`, hostname) envdFileData, err := os.ReadFile(storage.HostEnvdPath) if err != nil { From 37b4b5b6272ac1fc259360dc15d6fa9b87112902 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 8 Sep 2025 18:43:06 -0700 Subject: [PATCH 13/27] use new mmds event address field in envd and replace any pre-exsiting entries in /etc/hosts --- packages/envd/internal/host/mmds.go | 24 +++++++--- packages/envd/internal/host/network.go | 44 +++++++++++++++++++ .../envd/internal/logs/exporter/exporter.go | 10 ++--- 3 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 packages/envd/internal/host/network.go diff --git a/packages/envd/internal/host/mmds.go b/packages/envd/internal/host/mmds.go index bb22592570..f0c895f913 100644 --- a/packages/envd/internal/host/mmds.go +++ b/packages/envd/internal/host/mmds.go @@ -22,17 +22,19 @@ const ( ) type MMDSOpts struct { - TraceID string `json:"traceID"` - InstanceID string `json:"instanceID"` - EnvID string `json:"envID"` - Address string `json:"address"` + TraceID string `json:"traceID"` + InstanceID string `json:"instanceID"` + EnvID string `json:"envID"` + LogsCollectorAddress string `json:"address"` + EventsAddress string `json:"eventsAddress"` } -func (opts *MMDSOpts) Update(traceID, instanceID, envID, address string) { +func (opts *MMDSOpts) Update(traceID, instanceID, envID, logsCollectorAddress, eventsAddress string) { opts.TraceID = traceID opts.InstanceID = instanceID opts.EnvID = envID - opts.Address = address + opts.LogsCollectorAddress = logsCollectorAddress + opts.EventsAddress = eventsAddress } func (opts *MMDSOpts) AddOptsToJSON(jsonLogs []byte) ([]byte, error) { @@ -138,6 +140,7 @@ func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *ut envVars.Store("E2B_SANDBOX_ID", mmdsOpts.InstanceID) envVars.Store("E2B_TEMPLATE_ID", mmdsOpts.EnvID) + envVars.Store("E2B_EVENTS_ADDRESS", mmdsOpts.EventsAddress) if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_SANDBOX_ID"), []byte(mmdsOpts.InstanceID), 0o666); err != nil { fmt.Fprintf(os.Stderr, "error writing sandbox ID file: %v\n", err) } @@ -145,9 +148,16 @@ func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *ut fmt.Fprintf(os.Stderr, "error writing template ID file: %v\n", err) } - if mmdsOpts.Address != "" { + if mmdsOpts.LogsCollectorAddress != "" { mmdsChan <- mmdsOpts } + + if mmdsOpts.EventsAddress != "" { + if err := AddEventsHostEntry(mmdsOpts.EventsAddress); err != nil { + fmt.Fprintf(os.Stderr, "error adding events host entry: %v\n", err) + } + } + return } } diff --git a/packages/envd/internal/host/network.go b/packages/envd/internal/host/network.go new file mode 100644 index 0000000000..847449625e --- /dev/null +++ b/packages/envd/internal/host/network.go @@ -0,0 +1,44 @@ +package host + +import ( + "fmt" + "os" + "strings" +) + +func AddEventsHostEntry(address string) error { + hostsEntry := fmt.Sprintf("%s events.e2b.dev", address) + // Read existing hosts file + content, err := os.ReadFile("/etc/hosts") + if err != nil { + return fmt.Errorf("failed to read /etc/hosts: %w", err) + } + + // Filter out any existing events.e2b.dev entries + lines := strings.Split(string(content), "\n") + filteredLines := make([]string, 0, len(lines)) + for _, line := range lines { + if line == "" { + continue + } + if !strings.Contains(line, "events.e2b.dev") { + filteredLines = append(filteredLines, line) + } + } + + // Add the new entry + filteredLines = append(filteredLines, hostsEntry) + + // Write back to file + f, err := os.OpenFile("/etc/hosts", os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("failed to open /etc/hosts: %w", err) + } + defer f.Close() + + if _, err := f.WriteString(strings.Join(filteredLines, "\n") + "\n"); err != nil { + return fmt.Errorf("failed to write to /etc/hosts: %w", err) + } + + return nil +} diff --git a/packages/envd/internal/logs/exporter/exporter.go b/packages/envd/internal/logs/exporter/exporter.go index a142d37c0c..12df911d83 100644 --- a/packages/envd/internal/logs/exporter/exporter.go +++ b/packages/envd/internal/logs/exporter/exporter.go @@ -37,9 +37,9 @@ func NewHTTPLogsExporter(ctx context.Context, isNotFC bool, mmdsChan <-chan *hos isNotFC: isNotFC, startOnce: sync.Once{}, mmdsOpts: &host.MMDSOpts{ - InstanceID: "unknown", - EnvID: "unknown", - Address: "", + InstanceID: "unknown", + EnvID: "unknown", + LogsCollectorAddress: "", }, } @@ -85,7 +85,7 @@ func (w *HTTPExporter) listenForMMDSOptsAndStart(ctx context.Context, mmdsChan < w.mmdsLock.Lock() w.mmdsOpts.Update( - mmdsOpts.TraceID, mmdsOpts.InstanceID, mmdsOpts.EnvID, mmdsOpts.Address) + mmdsOpts.TraceID, mmdsOpts.InstanceID, mmdsOpts.EnvID, mmdsOpts.LogsCollectorAddress, mmdsOpts.EventsAddress) w.mmdsLock.Unlock() w.startOnce.Do(func() { @@ -123,7 +123,7 @@ func (w *HTTPExporter) start(ctx context.Context) { continue } - err = w.sendInstanceLogs(ctx, logLineWithOpts, w.mmdsOpts.Address) + err = w.sendInstanceLogs(ctx, logLineWithOpts, w.mmdsOpts.LogsCollectorAddress) if err != nil { log.Printf("error sending instance logs: %+v", err) From 2834798d73a6dd834d1e7c12ab0f6406dfa70a2a Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 8 Sep 2025 18:53:00 -0700 Subject: [PATCH 14/27] update consts --- packages/orchestrator/internal/consts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go index a0c3f72414..62c73b4ed3 100644 --- a/packages/orchestrator/internal/consts.go +++ b/packages/orchestrator/internal/consts.go @@ -5,9 +5,9 @@ import ( ) const ( - // Private IP don't leave the sandbox through the network bridge, so we use a reserved IP address for it: - // See TEST-NET-3 on https://en.wikipedia.org/wiki/Reserved_IP_addresses - defaultSandboxEventIP = "203.0.113.0" + // Same range used by metadata services in Firecracker/EC2 + // https://en.m.wikipedia.org/wiki/Link-local_address + defaultSandboxEventIP = "169.254.0.7" defaultEventProxyPort = "5010" ) From 226cf068e4c59630f2aa772909f5f9761f205198 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 8 Sep 2025 19:30:23 -0700 Subject: [PATCH 15/27] revert to TEST-NET-3 ip; gofumpt --- packages/envd/internal/host/network.go | 2 +- packages/orchestrator/internal/consts.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/envd/internal/host/network.go b/packages/envd/internal/host/network.go index 847449625e..437b91542f 100644 --- a/packages/envd/internal/host/network.go +++ b/packages/envd/internal/host/network.go @@ -30,7 +30,7 @@ func AddEventsHostEntry(address string) error { filteredLines = append(filteredLines, hostsEntry) // Write back to file - f, err := os.OpenFile("/etc/hosts", os.O_WRONLY|os.O_TRUNC, 0644) + f, err := os.OpenFile("/etc/hosts", os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return fmt.Errorf("failed to open /etc/hosts: %w", err) } diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go index 62c73b4ed3..a0c3f72414 100644 --- a/packages/orchestrator/internal/consts.go +++ b/packages/orchestrator/internal/consts.go @@ -5,9 +5,9 @@ import ( ) const ( - // Same range used by metadata services in Firecracker/EC2 - // https://en.m.wikipedia.org/wiki/Link-local_address - defaultSandboxEventIP = "169.254.0.7" + // Private IP don't leave the sandbox through the network bridge, so we use a reserved IP address for it: + // See TEST-NET-3 on https://en.wikipedia.org/wiki/Reserved_IP_addresses + defaultSandboxEventIP = "203.0.113.0" defaultEventProxyPort = "5010" ) From 5a163511af764c53f67394db2f4a29565307c4e7 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 13:39:31 -0700 Subject: [PATCH 16/27] remove changes from setup-networking-in-sandbox-for-events-endpoint-e2b-2961 --- packages/envd/internal/host/mmds.go | 164 ----------------- packages/envd/internal/host/network.go | 44 ----- .../envd/internal/logs/exporter/exporter.go | 173 ------------------ .../orchestrator/internal/sandbox/fc/mmds.go | 1 - .../internal/sandbox/network/network.go | 12 -- .../internal/sandbox/network/slot.go | 2 +- 6 files changed, 1 insertion(+), 395 deletions(-) delete mode 100644 packages/envd/internal/host/mmds.go delete mode 100644 packages/envd/internal/host/network.go delete mode 100644 packages/envd/internal/logs/exporter/exporter.go diff --git a/packages/envd/internal/host/mmds.go b/packages/envd/internal/host/mmds.go deleted file mode 100644 index f0c895f913..0000000000 --- a/packages/envd/internal/host/mmds.go +++ /dev/null @@ -1,164 +0,0 @@ -package host - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "time" - - "github.com/e2b-dev/infra/packages/envd/internal/utils" -) - -const ( - E2BRunDir = "/run/e2b" // store sandbox metadata files here - - mmdsDefaultAddress = "169.254.169.254" - mmdsTokenExpiration = 60 * time.Second -) - -type MMDSOpts struct { - TraceID string `json:"traceID"` - InstanceID string `json:"instanceID"` - EnvID string `json:"envID"` - LogsCollectorAddress string `json:"address"` - EventsAddress string `json:"eventsAddress"` -} - -func (opts *MMDSOpts) Update(traceID, instanceID, envID, logsCollectorAddress, eventsAddress string) { - opts.TraceID = traceID - opts.InstanceID = instanceID - opts.EnvID = envID - opts.LogsCollectorAddress = logsCollectorAddress - opts.EventsAddress = eventsAddress -} - -func (opts *MMDSOpts) AddOptsToJSON(jsonLogs []byte) ([]byte, error) { - parsed := make(map[string]interface{}) - - err := json.Unmarshal(jsonLogs, &parsed) - if err != nil { - return nil, err - } - - parsed["instanceID"] = opts.InstanceID - parsed["envID"] = opts.EnvID - parsed["traceID"] = opts.TraceID - - data, err := json.Marshal(parsed) - - return data, err -} - -func getMMDSToken(ctx context.Context, client *http.Client) (string, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://"+mmdsDefaultAddress+"/latest/api/token", new(bytes.Buffer)) - if err != nil { - return "", err - } - - request.Header["X-metadata-token-ttl-seconds"] = []string{fmt.Sprint(mmdsTokenExpiration.Seconds())} - - response, err := client.Do(request) - if err != nil { - return "", err - } - defer response.Body.Close() - - body, err := io.ReadAll(response.Body) - if err != nil { - return "", err - } - - token := string(body) - - if len(token) == 0 { - return "", fmt.Errorf("mmds token is an empty string") - } - - return token, nil -} - -func getMMDSOpts(ctx context.Context, client *http.Client, token string) (*MMDSOpts, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+mmdsDefaultAddress, new(bytes.Buffer)) - if err != nil { - return nil, err - } - - request.Header["X-metadata-token"] = []string{token} - request.Header["Accept"] = []string{"application/json"} - - response, err := client.Do(request) - if err != nil { - return nil, err - } - - defer response.Body.Close() - - body, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - var opts MMDSOpts - - err = json.Unmarshal(body, &opts) - if err != nil { - return nil, err - } - - return &opts, nil -} - -func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *utils.Map[string, string]) { - httpClient := &http.Client{} - defer httpClient.CloseIdleConnections() - - ticker := time.NewTicker(50 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - fmt.Fprintf(os.Stderr, "context cancelled while waiting for mmds opts") - return - case <-ticker.C: - token, err := getMMDSToken(ctx, httpClient) - if err != nil { - fmt.Fprintf(os.Stderr, "error getting mmds token: %v\n", err) - continue - } - - mmdsOpts, err := getMMDSOpts(ctx, httpClient, token) - if err != nil { - fmt.Fprintf(os.Stderr, "error getting mmds opts: %v\n", err) - continue - } - - envVars.Store("E2B_SANDBOX_ID", mmdsOpts.InstanceID) - envVars.Store("E2B_TEMPLATE_ID", mmdsOpts.EnvID) - envVars.Store("E2B_EVENTS_ADDRESS", mmdsOpts.EventsAddress) - if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_SANDBOX_ID"), []byte(mmdsOpts.InstanceID), 0o666); err != nil { - fmt.Fprintf(os.Stderr, "error writing sandbox ID file: %v\n", err) - } - if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_TEMPLATE_ID"), []byte(mmdsOpts.EnvID), 0o666); err != nil { - fmt.Fprintf(os.Stderr, "error writing template ID file: %v\n", err) - } - - if mmdsOpts.LogsCollectorAddress != "" { - mmdsChan <- mmdsOpts - } - - if mmdsOpts.EventsAddress != "" { - if err := AddEventsHostEntry(mmdsOpts.EventsAddress); err != nil { - fmt.Fprintf(os.Stderr, "error adding events host entry: %v\n", err) - } - } - - return - } - } -} diff --git a/packages/envd/internal/host/network.go b/packages/envd/internal/host/network.go deleted file mode 100644 index 437b91542f..0000000000 --- a/packages/envd/internal/host/network.go +++ /dev/null @@ -1,44 +0,0 @@ -package host - -import ( - "fmt" - "os" - "strings" -) - -func AddEventsHostEntry(address string) error { - hostsEntry := fmt.Sprintf("%s events.e2b.dev", address) - // Read existing hosts file - content, err := os.ReadFile("/etc/hosts") - if err != nil { - return fmt.Errorf("failed to read /etc/hosts: %w", err) - } - - // Filter out any existing events.e2b.dev entries - lines := strings.Split(string(content), "\n") - filteredLines := make([]string, 0, len(lines)) - for _, line := range lines { - if line == "" { - continue - } - if !strings.Contains(line, "events.e2b.dev") { - filteredLines = append(filteredLines, line) - } - } - - // Add the new entry - filteredLines = append(filteredLines, hostsEntry) - - // Write back to file - f, err := os.OpenFile("/etc/hosts", os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { - return fmt.Errorf("failed to open /etc/hosts: %w", err) - } - defer f.Close() - - if _, err := f.WriteString(strings.Join(filteredLines, "\n") + "\n"); err != nil { - return fmt.Errorf("failed to write to /etc/hosts: %w", err) - } - - return nil -} diff --git a/packages/envd/internal/logs/exporter/exporter.go b/packages/envd/internal/logs/exporter/exporter.go deleted file mode 100644 index 12df911d83..0000000000 --- a/packages/envd/internal/logs/exporter/exporter.go +++ /dev/null @@ -1,173 +0,0 @@ -package exporter - -import ( - "bytes" - "context" - "fmt" - "log" - "net/http" - "os" - "sync" - "time" - - "github.com/e2b-dev/infra/packages/envd/internal/host" -) - -const ExporterTimeout = 10 * time.Second - -type HTTPExporter struct { - client http.Client - logs [][]byte - isNotFC bool - mmdsOpts *host.MMDSOpts - - // Concurrency coordination - triggers chan struct{} - logLock sync.RWMutex - mmdsLock sync.RWMutex - startOnce sync.Once -} - -func NewHTTPLogsExporter(ctx context.Context, isNotFC bool, mmdsChan <-chan *host.MMDSOpts) *HTTPExporter { - exporter := &HTTPExporter{ - client: http.Client{ - Timeout: ExporterTimeout, - }, - triggers: make(chan struct{}, 1), - isNotFC: isNotFC, - startOnce: sync.Once{}, - mmdsOpts: &host.MMDSOpts{ - InstanceID: "unknown", - EnvID: "unknown", - LogsCollectorAddress: "", - }, - } - - go exporter.listenForMMDSOptsAndStart(ctx, mmdsChan) - - return exporter -} - -func (w *HTTPExporter) sendInstanceLogs(ctx context.Context, logs []byte, address string) error { - if address == "" { - return nil - } - - request, err := http.NewRequestWithContext(ctx, http.MethodPost, address, bytes.NewBuffer(logs)) - if err != nil { - return err - } - - request.Header.Set("Content-Type", "application/json") - - response, err := w.client.Do(request) - if err != nil { - return err - } - defer response.Body.Close() - - return nil -} - -func printLog(logs []byte) { - fmt.Fprintf(os.Stdout, "%v", string(logs)) -} - -func (w *HTTPExporter) listenForMMDSOptsAndStart(ctx context.Context, mmdsChan <-chan *host.MMDSOpts) { - for { - select { - case <-ctx.Done(): - return - case mmdsOpts, ok := <-mmdsChan: - if !ok { - return - } - - w.mmdsLock.Lock() - w.mmdsOpts.Update( - mmdsOpts.TraceID, mmdsOpts.InstanceID, mmdsOpts.EnvID, mmdsOpts.LogsCollectorAddress, mmdsOpts.EventsAddress) - w.mmdsLock.Unlock() - - w.startOnce.Do(func() { - w.start(ctx) - }) - } - } -} - -func (w *HTTPExporter) start(ctx context.Context) { - for range w.triggers { - logs := w.getAllLogs() - - if len(logs) == 0 { - continue - } - - if w.isNotFC { - for _, log := range logs { - fmt.Fprintf(os.Stdout, "%v", string(log)) - } - - continue - } - - for _, logLine := range logs { - w.mmdsLock.RLock() - logLineWithOpts, err := w.mmdsOpts.AddOptsToJSON(logLine) - w.mmdsLock.RUnlock() - if err != nil { - log.Printf("error adding instance logging options (%+v) to JSON (%+v) with logs : %v\n", w.mmdsOpts, logLine, err) - - printLog(logLine) - - continue - } - - err = w.sendInstanceLogs(ctx, logLineWithOpts, w.mmdsOpts.LogsCollectorAddress) - if err != nil { - log.Printf("error sending instance logs: %+v", err) - - printLog(logLine) - - continue - } - } - } -} - -func (w *HTTPExporter) resumeProcessing() { - select { - case w.triggers <- struct{}{}: - default: - // Exporter processing already triggered - // This is expected behavior if the exporter is already processing logs - } -} - -func (w *HTTPExporter) Write(logs []byte) (int, error) { - logsCopy := make([]byte, len(logs)) - copy(logsCopy, logs) - - go w.addLogs(logsCopy) - - return len(logs), nil -} - -func (w *HTTPExporter) getAllLogs() [][]byte { - w.logLock.Lock() - defer w.logLock.Unlock() - - logs := w.logs - w.logs = nil - - return logs -} - -func (w *HTTPExporter) addLogs(logs []byte) { - w.logLock.Lock() - defer w.logLock.Unlock() - - w.logs = append(w.logs, logs) - - w.resumeProcessing() -} diff --git a/packages/orchestrator/internal/sandbox/fc/mmds.go b/packages/orchestrator/internal/sandbox/fc/mmds.go index e97135a99b..859dd477bf 100644 --- a/packages/orchestrator/internal/sandbox/fc/mmds.go +++ b/packages/orchestrator/internal/sandbox/fc/mmds.go @@ -11,7 +11,6 @@ type MmdsMetadata struct { LogsCollectorAddress string `json:"address"` TraceId string `json:"traceID"` TeamId string `json:"teamID"` - EventsAddress string `json:"eventsAddress"` } func (mm MmdsMetadata) LoggerMetadata() sbxlogger.SandboxMetadata { diff --git a/packages/orchestrator/internal/sandbox/network/network.go b/packages/orchestrator/internal/sandbox/network/network.go index 6b0eba4f29..6fadb92ed2 100644 --- a/packages/orchestrator/internal/sandbox/network/network.go +++ b/packages/orchestrator/internal/sandbox/network/network.go @@ -10,8 +10,6 @@ import ( "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "go.uber.org/zap" - - consts "github.com/e2b-dev/infra/packages/orchestrator/internal" ) func (s *Slot) CreateNetwork() error { @@ -216,16 +214,6 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } - // Redirect traffic destined for event proxy server - err = tables.Append( - "nat", "PREROUTING", "-i", s.VethName(), - "-p", "tcp", "-d", consts.GetSandboxEventIP(), "--dport", "80", - "-j", "REDIRECT", "--to-port", consts.GetEventProxyPort(), - ) - if err != nil { - return fmt.Errorf("error creating HTTP redirect rule to sandbox event proxy server: %w", err) - } - return nil } diff --git a/packages/orchestrator/internal/sandbox/network/slot.go b/packages/orchestrator/internal/sandbox/network/slot.go index ad888ec287..a85d3f0a01 100644 --- a/packages/orchestrator/internal/sandbox/network/slot.go +++ b/packages/orchestrator/internal/sandbox/network/slot.go @@ -8,11 +8,11 @@ import ( "path/filepath" "sync/atomic" - "github.com/containernetworking/plugins/pkg/ns" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" netutils "k8s.io/utils/net" + "github.com/containernetworking/plugins/pkg/ns" "github.com/e2b-dev/infra/packages/shared/pkg/env" ) From 4d3c870e81b5173c8568d107507d691f392be1e4 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 13:40:22 -0700 Subject: [PATCH 17/27] remove eventaddress from mmds --- packages/orchestrator/internal/sandbox/sandbox.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index 1dd90c7236..e3bb2f7f21 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -13,7 +13,6 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/orchestrator/internal" globalconfig "github.com/e2b-dev/infra/packages/orchestrator/internal/config" "github.com/e2b-dev/infra/packages/orchestrator/internal/events" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" @@ -460,7 +459,6 @@ func ResumeSandbox( LogsCollectorAddress: fmt.Sprintf("http://%s", logsCollectorIP), TraceId: traceID, TeamId: runtime.TeamID, - EventsAddress: internal.GetSandboxEventIP(), }, fcUffdPath, snapfile, From 736edceace92a23ebf27a643122d989160afb869 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 13:47:31 -0700 Subject: [PATCH 18/27] make fmt --- packages/orchestrator/go.mod | 2 +- packages/orchestrator/go.sum | 2 ++ packages/orchestrator/internal/sandbox/network/slot.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index 4f50718896..f4aca117a2 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -37,7 +37,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.36.0 go.opentelemetry.io/otel/trace v1.36.0 go.uber.org/zap v1.27.0 - golang.org/x/sync v0.15.0 + golang.org/x/sync v0.17.0 golang.org/x/sys v0.33.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 062df6daa6..2fa1f01f97 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1303,6 +1303,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/packages/orchestrator/internal/sandbox/network/slot.go b/packages/orchestrator/internal/sandbox/network/slot.go index a85d3f0a01..ad888ec287 100644 --- a/packages/orchestrator/internal/sandbox/network/slot.go +++ b/packages/orchestrator/internal/sandbox/network/slot.go @@ -8,11 +8,11 @@ import ( "path/filepath" "sync/atomic" + "github.com/containernetworking/plugins/pkg/ns" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" netutils "k8s.io/utils/net" - "github.com/containernetworking/plugins/pkg/ns" "github.com/e2b-dev/infra/packages/shared/pkg/env" ) From 870168a26667683a56d42d3322c0c895ac4b9139 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 13:52:54 -0700 Subject: [PATCH 19/27] add missing envd files --- packages/envd/internal/host/mmds.go | 154 ++++++++++++++++ .../envd/internal/logs/exporter/exporter.go | 173 ++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 packages/envd/internal/host/mmds.go create mode 100644 packages/envd/internal/logs/exporter/exporter.go diff --git a/packages/envd/internal/host/mmds.go b/packages/envd/internal/host/mmds.go new file mode 100644 index 0000000000..bb22592570 --- /dev/null +++ b/packages/envd/internal/host/mmds.go @@ -0,0 +1,154 @@ +package host + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/e2b-dev/infra/packages/envd/internal/utils" +) + +const ( + E2BRunDir = "/run/e2b" // store sandbox metadata files here + + mmdsDefaultAddress = "169.254.169.254" + mmdsTokenExpiration = 60 * time.Second +) + +type MMDSOpts struct { + TraceID string `json:"traceID"` + InstanceID string `json:"instanceID"` + EnvID string `json:"envID"` + Address string `json:"address"` +} + +func (opts *MMDSOpts) Update(traceID, instanceID, envID, address string) { + opts.TraceID = traceID + opts.InstanceID = instanceID + opts.EnvID = envID + opts.Address = address +} + +func (opts *MMDSOpts) AddOptsToJSON(jsonLogs []byte) ([]byte, error) { + parsed := make(map[string]interface{}) + + err := json.Unmarshal(jsonLogs, &parsed) + if err != nil { + return nil, err + } + + parsed["instanceID"] = opts.InstanceID + parsed["envID"] = opts.EnvID + parsed["traceID"] = opts.TraceID + + data, err := json.Marshal(parsed) + + return data, err +} + +func getMMDSToken(ctx context.Context, client *http.Client) (string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://"+mmdsDefaultAddress+"/latest/api/token", new(bytes.Buffer)) + if err != nil { + return "", err + } + + request.Header["X-metadata-token-ttl-seconds"] = []string{fmt.Sprint(mmdsTokenExpiration.Seconds())} + + response, err := client.Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return "", err + } + + token := string(body) + + if len(token) == 0 { + return "", fmt.Errorf("mmds token is an empty string") + } + + return token, nil +} + +func getMMDSOpts(ctx context.Context, client *http.Client, token string) (*MMDSOpts, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+mmdsDefaultAddress, new(bytes.Buffer)) + if err != nil { + return nil, err + } + + request.Header["X-metadata-token"] = []string{token} + request.Header["Accept"] = []string{"application/json"} + + response, err := client.Do(request) + if err != nil { + return nil, err + } + + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + var opts MMDSOpts + + err = json.Unmarshal(body, &opts) + if err != nil { + return nil, err + } + + return &opts, nil +} + +func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *utils.Map[string, string]) { + httpClient := &http.Client{} + defer httpClient.CloseIdleConnections() + + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + fmt.Fprintf(os.Stderr, "context cancelled while waiting for mmds opts") + return + case <-ticker.C: + token, err := getMMDSToken(ctx, httpClient) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting mmds token: %v\n", err) + continue + } + + mmdsOpts, err := getMMDSOpts(ctx, httpClient, token) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting mmds opts: %v\n", err) + continue + } + + envVars.Store("E2B_SANDBOX_ID", mmdsOpts.InstanceID) + envVars.Store("E2B_TEMPLATE_ID", mmdsOpts.EnvID) + if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_SANDBOX_ID"), []byte(mmdsOpts.InstanceID), 0o666); err != nil { + fmt.Fprintf(os.Stderr, "error writing sandbox ID file: %v\n", err) + } + if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_TEMPLATE_ID"), []byte(mmdsOpts.EnvID), 0o666); err != nil { + fmt.Fprintf(os.Stderr, "error writing template ID file: %v\n", err) + } + + if mmdsOpts.Address != "" { + mmdsChan <- mmdsOpts + } + return + } + } +} diff --git a/packages/envd/internal/logs/exporter/exporter.go b/packages/envd/internal/logs/exporter/exporter.go new file mode 100644 index 0000000000..a142d37c0c --- /dev/null +++ b/packages/envd/internal/logs/exporter/exporter.go @@ -0,0 +1,173 @@ +package exporter + +import ( + "bytes" + "context" + "fmt" + "log" + "net/http" + "os" + "sync" + "time" + + "github.com/e2b-dev/infra/packages/envd/internal/host" +) + +const ExporterTimeout = 10 * time.Second + +type HTTPExporter struct { + client http.Client + logs [][]byte + isNotFC bool + mmdsOpts *host.MMDSOpts + + // Concurrency coordination + triggers chan struct{} + logLock sync.RWMutex + mmdsLock sync.RWMutex + startOnce sync.Once +} + +func NewHTTPLogsExporter(ctx context.Context, isNotFC bool, mmdsChan <-chan *host.MMDSOpts) *HTTPExporter { + exporter := &HTTPExporter{ + client: http.Client{ + Timeout: ExporterTimeout, + }, + triggers: make(chan struct{}, 1), + isNotFC: isNotFC, + startOnce: sync.Once{}, + mmdsOpts: &host.MMDSOpts{ + InstanceID: "unknown", + EnvID: "unknown", + Address: "", + }, + } + + go exporter.listenForMMDSOptsAndStart(ctx, mmdsChan) + + return exporter +} + +func (w *HTTPExporter) sendInstanceLogs(ctx context.Context, logs []byte, address string) error { + if address == "" { + return nil + } + + request, err := http.NewRequestWithContext(ctx, http.MethodPost, address, bytes.NewBuffer(logs)) + if err != nil { + return err + } + + request.Header.Set("Content-Type", "application/json") + + response, err := w.client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + + return nil +} + +func printLog(logs []byte) { + fmt.Fprintf(os.Stdout, "%v", string(logs)) +} + +func (w *HTTPExporter) listenForMMDSOptsAndStart(ctx context.Context, mmdsChan <-chan *host.MMDSOpts) { + for { + select { + case <-ctx.Done(): + return + case mmdsOpts, ok := <-mmdsChan: + if !ok { + return + } + + w.mmdsLock.Lock() + w.mmdsOpts.Update( + mmdsOpts.TraceID, mmdsOpts.InstanceID, mmdsOpts.EnvID, mmdsOpts.Address) + w.mmdsLock.Unlock() + + w.startOnce.Do(func() { + w.start(ctx) + }) + } + } +} + +func (w *HTTPExporter) start(ctx context.Context) { + for range w.triggers { + logs := w.getAllLogs() + + if len(logs) == 0 { + continue + } + + if w.isNotFC { + for _, log := range logs { + fmt.Fprintf(os.Stdout, "%v", string(log)) + } + + continue + } + + for _, logLine := range logs { + w.mmdsLock.RLock() + logLineWithOpts, err := w.mmdsOpts.AddOptsToJSON(logLine) + w.mmdsLock.RUnlock() + if err != nil { + log.Printf("error adding instance logging options (%+v) to JSON (%+v) with logs : %v\n", w.mmdsOpts, logLine, err) + + printLog(logLine) + + continue + } + + err = w.sendInstanceLogs(ctx, logLineWithOpts, w.mmdsOpts.Address) + if err != nil { + log.Printf("error sending instance logs: %+v", err) + + printLog(logLine) + + continue + } + } + } +} + +func (w *HTTPExporter) resumeProcessing() { + select { + case w.triggers <- struct{}{}: + default: + // Exporter processing already triggered + // This is expected behavior if the exporter is already processing logs + } +} + +func (w *HTTPExporter) Write(logs []byte) (int, error) { + logsCopy := make([]byte, len(logs)) + copy(logsCopy, logs) + + go w.addLogs(logsCopy) + + return len(logs), nil +} + +func (w *HTTPExporter) getAllLogs() [][]byte { + w.logLock.Lock() + defer w.logLock.Unlock() + + logs := w.logs + w.logs = nil + + return logs +} + +func (w *HTTPExporter) addLogs(logs []byte) { + w.logLock.Lock() + defer w.logLock.Unlock() + + w.logs = append(w.logs, logs) + + w.resumeProcessing() +} From 91373c30ad0628398901ae46cb97187aa3d4997c Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 13:56:04 -0700 Subject: [PATCH 20/27] make tidy --- packages/api/go.mod | 2 +- packages/api/go.sum | 3 +-- packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 3 +-- packages/db/go.mod | 2 +- packages/db/go.sum | 3 +-- packages/docker-reverse-proxy/go.mod | 2 +- packages/docker-reverse-proxy/go.sum | 3 +-- packages/envd/go.mod | 2 +- packages/envd/go.sum | 3 +-- packages/orchestrator/go.sum | 2 -- packages/shared/go.mod | 2 +- packages/shared/go.sum | 3 +-- 13 files changed, 12 insertions(+), 20 deletions(-) diff --git a/packages/api/go.mod b/packages/api/go.mod index 06449bdfb3..26ce023de8 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -253,7 +253,7 @@ require ( golang.org/x/mod v0.25.0 // indirect golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.15.0 + golang.org/x/sync v0.17.0 golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.11.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index 5c7574c925..6dfcddce57 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -1178,8 +1178,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 43566a03da..d4759dbc16 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -94,7 +94,7 @@ require ( golang.org/x/crypto v0.39.0 // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect + golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.26.0 // indirect golang.org/x/tools v0.34.0 // indirect diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index 47c170b0c0..fcd3861abb 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -931,8 +931,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/packages/db/go.mod b/packages/db/go.mod index cec485cf80..d89ba6af8d 100644 --- a/packages/db/go.mod +++ b/packages/db/go.mod @@ -81,7 +81,7 @@ require ( golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect + golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.26.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect diff --git a/packages/db/go.sum b/packages/db/go.sum index 6051654469..2db05c50c6 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -326,8 +326,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/packages/docker-reverse-proxy/go.mod b/packages/docker-reverse-proxy/go.mod index 2944e6d418..b78836f48e 100644 --- a/packages/docker-reverse-proxy/go.mod +++ b/packages/docker-reverse-proxy/go.mod @@ -26,6 +26,6 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/zclconf/go-cty v1.14.1 // indirect golang.org/x/mod v0.25.0 // indirect - golang.org/x/sync v0.15.0 // indirect + golang.org/x/sync v0.17.0 // indirect golang.org/x/text v0.26.0 // indirect ) diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index 9e06dd421e..0ed5a774ff 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -53,8 +53,7 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/packages/envd/go.mod b/packages/envd/go.mod index 1f141c07a4..795936dd05 100644 --- a/packages/envd/go.mod +++ b/packages/envd/go.mod @@ -52,7 +52,7 @@ require ( github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/mod v0.25.0 // indirect - golang.org/x/sync v0.15.0 // indirect + golang.org/x/sync v0.17.0 // indirect golang.org/x/text v0.26.0 // indirect golang.org/x/tools v0.34.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/packages/envd/go.sum b/packages/envd/go.sum index 63ce83da6d..f3030653b6 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -176,8 +176,7 @@ golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 2fa1f01f97..4bd109ab3c 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -1301,8 +1301,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/packages/shared/go.mod b/packages/shared/go.mod index f6c4f2b887..3d3b4c59a3 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -49,7 +49,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/mod v0.25.0 golang.org/x/oauth2 v0.30.0 - golang.org/x/sync v0.15.0 + golang.org/x/sync v0.17.0 google.golang.org/api v0.214.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 diff --git a/packages/shared/go.sum b/packages/shared/go.sum index 4e47d8b144..7e51c83117 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1070,8 +1070,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From efe9dd6842cc23899e46bbaa4c6119ca83da04f0 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 13:59:21 -0700 Subject: [PATCH 21/27] go mod tidy in integration --- tests/integration/go.mod | 2 +- tests/integration/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/go.mod b/tests/integration/go.mod index 5cf5f78cb8..de11dde3ba 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -14,7 +14,7 @@ require ( github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.10.0 golang.org/x/net v0.41.0 - golang.org/x/sync v0.15.0 + golang.org/x/sync v0.17.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 ) diff --git a/tests/integration/go.sum b/tests/integration/go.sum index f0bfc40326..2118caf6e3 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -259,8 +259,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 7a8626424702b88af8810dc158ef296ee754204e Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 9 Sep 2025 20:01:46 -0700 Subject: [PATCH 22/27] make tidy --- packages/api/go.sum | 1 + packages/client-proxy/go.sum | 1 + packages/db/go.sum | 1 + packages/docker-reverse-proxy/go.sum | 1 + packages/envd/go.sum | 1 + packages/shared/go.sum | 1 + tests/integration/go.mod | 2 +- tests/integration/go.sum | 4 ++-- 8 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/api/go.sum b/packages/api/go.sum index 6dfcddce57..eac09f3e98 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -1179,6 +1179,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index fcd3861abb..4681e6fe24 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -932,6 +932,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/packages/db/go.sum b/packages/db/go.sum index 2db05c50c6..0db3fb3cad 100644 --- a/packages/db/go.sum +++ b/packages/db/go.sum @@ -327,6 +327,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/packages/docker-reverse-proxy/go.sum b/packages/docker-reverse-proxy/go.sum index 0ed5a774ff..9f328a39b5 100644 --- a/packages/docker-reverse-proxy/go.sum +++ b/packages/docker-reverse-proxy/go.sum @@ -54,6 +54,7 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/packages/envd/go.sum b/packages/envd/go.sum index f3030653b6..ce28523776 100644 --- a/packages/envd/go.sum +++ b/packages/envd/go.sum @@ -177,6 +177,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/packages/shared/go.sum b/packages/shared/go.sum index 7e51c83117..5bf6cf9df6 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -1071,6 +1071,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/tests/integration/go.mod b/tests/integration/go.mod index 5cf5f78cb8..de11dde3ba 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -14,7 +14,7 @@ require ( github.com/oapi-codegen/runtime v1.1.1 github.com/stretchr/testify v1.10.0 golang.org/x/net v0.41.0 - golang.org/x/sync v0.15.0 + golang.org/x/sync v0.17.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 ) diff --git a/tests/integration/go.sum b/tests/integration/go.sum index f0bfc40326..2118caf6e3 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -259,8 +259,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From ba098014748f99f96d8f2abf4c4d2e6c1d635b38 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 11 Sep 2025 12:31:40 -0700 Subject: [PATCH 23/27] return error if redis-client is nil in ip setter --- packages/orchestrator/internal/events/sandbox_events_store.go | 4 ++++ packages/orchestrator/internal/sandbox/sandbox.go | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/internal/events/sandbox_events_store.go b/packages/orchestrator/internal/events/sandbox_events_store.go index d947fa9eed..aae34ba6eb 100644 --- a/packages/orchestrator/internal/events/sandbox_events_store.go +++ b/packages/orchestrator/internal/events/sandbox_events_store.go @@ -3,6 +3,7 @@ package events import ( "context" "encoding/json" + "fmt" "time" "github.com/redis/go-redis/v9" @@ -51,6 +52,9 @@ func NewSandboxEventStore(redisClient redis.UniversalClient) SandboxEventStore { } func (c *sandboxEventStore) SetSandboxIP(ctx context.Context, sandboxID string, sandboxIP string) error { + if c.redisClient == nil { + return fmt.Errorf("redisClient is nil") + } return c.redisClient.Set(ctx, IPPrefix+sandboxIP, sandboxID, cacheTL).Err() } diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index de87280949..d93ac3afd5 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -517,7 +517,6 @@ func ResumeSandbox( if eventStore != nil { sandboxIP := ips.slot.HostIPString() - ctx = context.WithoutCancel(ctx) err = eventStore.SetSandboxIP(context.WithoutCancel(ctx), runtime.SandboxID, sandboxIP) if err != nil { // soft fail to not block this critical path From 30b0af36408e74745faebdea70ce0d500a0b4ccb Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 11 Sep 2025 12:55:30 -0700 Subject: [PATCH 24/27] add redis url to integration tests env vars --- tests/integration/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/Makefile b/tests/integration/Makefile index 77ee020ecb..9a45817f95 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -36,6 +36,8 @@ seed: test: test/. test/%: @export POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING); \ + export REDIS_URL=$(REDIS_URL); \ + export REDIS_CLUSTER_URL=$(REDIS_CLUSTER_URL); \ export TESTS_API_SERVER_URL=$(TESTS_API_SERVER_URL); \ export TESTS_ORCHESTRATOR_HOST=$(TESTS_ORCHESTRATOR_HOST); \ export TESTS_ENVD_PROXY=$(TESTS_ENVD_PROXY); \ From 93c674a0dc8b6b147daf0cc2686cd10ad16a7329 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 11 Sep 2025 13:24:28 -0700 Subject: [PATCH 25/27] setup redis in start-services action --- .github/actions/start-services/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 4b86e072d4..d4bad962df 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -75,12 +75,20 @@ runs: make -C packages/clickhouse migrate-without-build shell: bash + - name: Run Redis + run: | + echo "REDIS_URL=localhost:6379" >> .env.test + + docker run -d --name redis -p 6379:6379 redis:7.4.2-alpine + shell: bash + - name: Start Services env: ENVD_TIMEOUT: "60s" ORCHESTRATOR_SERVICES: "orchestrator,template-manager" SANDBOX_ACCESS_TOKEN_HASH_SEED: "abcdefghijklmnopqrstuvwxyz" TEMPLATE_MANAGER_HOST: "localhost:5008" + REDIS_URL: "localhost:6379" ARTIFACTS_REGISTRY_PROVIDER: "Local" STORAGE_PROVIDER: "Local" ENVIRONMENT: "local" From 54833de15b837503103a43724518428526493016 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 11 Sep 2025 15:25:13 -0700 Subject: [PATCH 26/27] add noop impl. of event interfaces --- .../events/mock_sandbox_events_proxy.go | 22 ++++++++ .../events/mock_sandbox_events_service.go | 23 ++++++++ .../events/mock_sandbox_events_store.go | 53 +++++++++++++++++++ .../internal/events/sandbox_events_proxy.go | 7 ++- packages/orchestrator/main.go | 21 +++++--- 5 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 packages/orchestrator/internal/events/mock_sandbox_events_proxy.go create mode 100644 packages/orchestrator/internal/events/mock_sandbox_events_service.go create mode 100644 packages/orchestrator/internal/events/mock_sandbox_events_store.go diff --git a/packages/orchestrator/internal/events/mock_sandbox_events_proxy.go b/packages/orchestrator/internal/events/mock_sandbox_events_proxy.go new file mode 100644 index 0000000000..229a31a8d2 --- /dev/null +++ b/packages/orchestrator/internal/events/mock_sandbox_events_proxy.go @@ -0,0 +1,22 @@ +package events + +import ( + "context" +) + +// NoopSandboxEventProxy is a simple Noop that doesn't do anything +type NoopSandboxEventProxy struct{} + +func NewNoopSandboxEventProxy() *NoopSandboxEventProxy { + return &NoopSandboxEventProxy{} +} + +func (m *NoopSandboxEventProxy) Start() error { + // No-op + return nil +} + +func (m *NoopSandboxEventProxy) Close(ctx context.Context) error { + // No-op + return nil +} diff --git a/packages/orchestrator/internal/events/mock_sandbox_events_service.go b/packages/orchestrator/internal/events/mock_sandbox_events_service.go new file mode 100644 index 0000000000..32a18acef8 --- /dev/null +++ b/packages/orchestrator/internal/events/mock_sandbox_events_service.go @@ -0,0 +1,23 @@ +package events + +import ( + "context" + + "github.com/e2b-dev/infra/packages/shared/pkg/events/event" +) + +// NoopSandboxEventsService is a simple Noop that doesn't do anything +type NoopSandboxEventsService struct{} + +func NewNoopSandboxEventsService() *NoopSandboxEventsService { + return &NoopSandboxEventsService{} +} + +func (m *NoopSandboxEventsService) HandleEvent(ctx context.Context, event event.SandboxEvent) { + // No-op +} + +func (m *NoopSandboxEventsService) Close(ctx context.Context) error { + // No-op + return nil +} diff --git a/packages/orchestrator/internal/events/mock_sandbox_events_store.go b/packages/orchestrator/internal/events/mock_sandbox_events_store.go new file mode 100644 index 0000000000..59759bc39a --- /dev/null +++ b/packages/orchestrator/internal/events/mock_sandbox_events_store.go @@ -0,0 +1,53 @@ +package events + +import ( + "context" + "time" +) + +// NoopSandboxEventStore is a simple Noop that doesn't do anything +type NoopSandboxEventStore struct{} + +func NewNoopSandboxEventStore() SandboxEventStore { + return &NoopSandboxEventStore{} +} + +func (m *NoopSandboxEventStore) SetSandboxIP(ctx context.Context, sandboxID string, sandboxIP string) error { + // No-op + return nil +} + +func (m *NoopSandboxEventStore) GetSandboxID(ctx context.Context, sandboxIP string) (string, error) { + // No-op + return "", nil +} + +func (m *NoopSandboxEventStore) DelSandboxIP(ctx context.Context, sandboxIP string) error { + // No-op + return nil +} + +func (m *NoopSandboxEventStore) GetLastEvent(ctx context.Context, sandboxID string) (*SandboxEvent, error) { + // No-op + return nil, nil +} + +func (m *NoopSandboxEventStore) GetLastNEvents(ctx context.Context, sandboxID string, n int) ([]*SandboxEvent, error) { + // No-op + return nil, nil +} + +func (m *NoopSandboxEventStore) AddEvent(ctx context.Context, sandboxID string, event *SandboxEvent, expiration time.Duration) error { + // No-op + return nil +} + +func (m *NoopSandboxEventStore) DelEvent(ctx context.Context, sandboxID string) error { + // No-op + return nil +} + +func (m *NoopSandboxEventStore) Close(ctx context.Context) error { + // No-op + return nil +} diff --git a/packages/orchestrator/internal/events/sandbox_events_proxy.go b/packages/orchestrator/internal/events/sandbox_events_proxy.go index 694d0c3ff9..271497d4cc 100644 --- a/packages/orchestrator/internal/events/sandbox_events_proxy.go +++ b/packages/orchestrator/internal/events/sandbox_events_proxy.go @@ -6,12 +6,17 @@ import ( "net/http" ) +type SandboxEventProxier interface { + Start() error + Close(ctx context.Context) error +} + // SandboxEventServer handles outbound HTTP requests from sandboxes calling the event.e2b.com endpoint type SandboxEventProxy struct { server *http.Server } -func NewSandboxEventProxy(port uint, store SandboxEventStore, handlers ...SandboxEventHandler) *SandboxEventProxy { +func NewSandboxEventProxy(port uint, store SandboxEventStore, handlers ...SandboxEventHandler) SandboxEventProxier { mux := http.NewServeMux() handlers = append(handlers, NewDefaultSandboxEventHandler(store)) diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index d6b9fbd7e1..540c4484f4 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -336,17 +336,24 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { zap.L().Info("Connected to Redis cluster") } - var redisPubSub pubsub.PubSub[event.SandboxEvent, webhooks.SandboxWebhooksMetaData] + var ( + redisPubSub pubsub.PubSub[event.SandboxEvent, webhooks.SandboxWebhooksMetaData] + sbxEventStore events.SandboxEventStore + sbxEventService events.EventService[event.SandboxEvent] + sbxEventProxy events.SandboxEventProxier + ) if redisClient != nil { redisPubSub = pubsub.NewRedisPubSub[event.SandboxEvent, webhooks.SandboxWebhooksMetaData](redisClient, "sandbox-webhooks") + sbxEventService = events.NewSandboxEventsService(featureFlags, redisPubSub, sandboxEventBatcher, globalLogger) + sbxEventStore = events.NewSandboxEventStore(redisClient) + sbxEventProxy = events.NewSandboxEventProxy(eventProxyPort, sbxEventStore) } else { redisPubSub = pubsub.NewMockPubSub[event.SandboxEvent, webhooks.SandboxWebhooksMetaData]() + sbxEventService = events.NewNoopSandboxEventsService() + sbxEventStore = events.NewNoopSandboxEventStore() + sbxEventProxy = events.NewNoopSandboxEventProxy() } - sbxEventService := events.NewSandboxEventsService(featureFlags, redisPubSub, sandboxEventBatcher, globalLogger) - sbxEventtore := events.NewSandboxEventStore(redisClient) - sbxEventProxy := events.NewSandboxEventProxy(eventProxyPort, sbxEventtore) - sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes) if err != nil { zap.L().Fatal("failed to create sandbox observer", zap.Error(err)) @@ -366,7 +373,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { Persistence: persistence, FeatureFlags: featureFlags, SbxEventService: sbxEventService, - SbxEventStore: sbxEventtore, + SbxEventStore: sbxEventStore, }, ) if err != nil { @@ -400,7 +407,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { sandboxObserver, limiter, sandboxEventBatcher, - sbxEventtore, + sbxEventStore, sbxEventProxy, ) From 4c96bffa92ee0939a24ca4a83bbc5544d319be0e Mon Sep 17 00:00:00 2001 From: 0div Date: Fri, 12 Sep 2025 15:10:23 -0700 Subject: [PATCH 27/27] remove unused pubsub --- packages/orchestrator/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 540c4484f4..078d911578 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -348,7 +348,6 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { sbxEventStore = events.NewSandboxEventStore(redisClient) sbxEventProxy = events.NewSandboxEventProxy(eventProxyPort, sbxEventStore) } else { - redisPubSub = pubsub.NewMockPubSub[event.SandboxEvent, webhooks.SandboxWebhooksMetaData]() sbxEventService = events.NewNoopSandboxEventsService() sbxEventStore = events.NewNoopSandboxEventStore() sbxEventProxy = events.NewNoopSandboxEventProxy()