From d8dc71f96b231c13e7095a7fcc152eee23ef1d1d Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 16 Jun 2025 11:52:39 -0700 Subject: [PATCH 01/21] WIP sandbox event endpoint boilerplate --- .../orchestrator/cmd/build-template/main.go | 1 + .../internal/proxy/event_proxy.go | 78 +++++++++++++++++++ .../internal/sandbox/network/network_linux.go | 6 ++ packages/orchestrator/internal/server/main.go | 1 + .../internal/template/build/rootfs.go | 4 +- packages/orchestrator/main.go | 35 +++++++-- 6 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 packages/orchestrator/internal/proxy/event_proxy.go diff --git a/packages/orchestrator/cmd/build-template/main.go b/packages/orchestrator/cmd/build-template/main.go index 6d2ded83d2..5e1ac533aa 100644 --- a/packages/orchestrator/cmd/build-template/main.go +++ b/packages/orchestrator/cmd/build-template/main.go @@ -30,6 +30,7 @@ import ( ) const proxyPort = 5007 +const eventProxyPort = 5010 func main() { ctx, cancel := context.WithCancel(context.Background()) diff --git a/packages/orchestrator/internal/proxy/event_proxy.go b/packages/orchestrator/internal/proxy/event_proxy.go new file mode 100644 index 0000000000..b1881a5188 --- /dev/null +++ b/packages/orchestrator/internal/proxy/event_proxy.go @@ -0,0 +1,78 @@ +package proxy + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + "go.uber.org/zap" +) + +// EventProxy handles outbound traffic from sandboxes calling the event.e2b.com domain +type EventProxy struct { + server *http.Server +} + +func NewEventProxy(port uint) *EventProxy { + server := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + } + + return &EventProxy{ + server: server, + } +} +func (p *EventProxy) Start() error { + serverTransport := &http.Transport{ + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 8192, + IdleConnTimeout: 620 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 24 * time.Hour, + DisableKeepAlives: true, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + zap.L().Info("Dialing", zap.String("network", network), zap.String("addr", addr)) + return net.Dial(network, addr) + }, + } + p.server.Handler = http.HandlerFunc(p.proxyHandler(serverTransport)) + + return p.server.ListenAndServeTLS("/etc/ssl/certs/cert.pem", "/etc/ssl/certs/key.pem") +} + +func (p *EventProxy) Close(ctx context.Context) error { + var err error + select { + case <-ctx.Done(): + err = p.server.Close() + default: + err = p.server.Shutdown(ctx) + } + if err != nil { + return fmt.Errorf("failed to shutdown proxy server: %w", err) + } + + return nil +} + +func (p *EventProxy) proxyHandler(transport *http.Transport) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + zap.L().Info("Forwarding request", zap.String("url", r.URL.String()), zap.String("method", r.Method)) + handleHTTP(w, r, transport) + } +} + +// handleHTTP handles regular HTTP requests +func handleHTTP(w http.ResponseWriter, r *http.Request, transport *http.Transport) { + zap.L().Info("[EVENT] handle HTTP request", zap.String("url", r.URL.String())) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, err := w.Write([]byte(`{"event_ack":true}`)) + if err != nil { + zap.L().Error("Failed to write response", zap.Error(err)) + } + return +} diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 4f3b2a333e..0fed8fce0e 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -217,6 +217,12 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } + // Redirect HTTP traffic destined for 10.20.7.67 to local prox + err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", "10.20.7.67", "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") + if err != nil { + return fmt.Errorf("error creating HTTP redirect rule to proxy: %w", err) + } + return nil } diff --git a/packages/orchestrator/internal/server/main.go b/packages/orchestrator/internal/server/main.go index dc0b57503b..0894c717f1 100644 --- a/packages/orchestrator/internal/server/main.go +++ b/packages/orchestrator/internal/server/main.go @@ -68,6 +68,7 @@ func New( tracer trace.Tracer, info *service.ServiceInfo, proxy *proxy.SandboxProxy, + eventProxy *proxy.EventProxy, sandboxes *smap.Map[*sandbox.Sandbox], ) (*Service, error) { srv := &Service{info: info} diff --git a/packages/orchestrator/internal/template/build/rootfs.go b/packages/orchestrator/internal/template/build/rootfs.go index 7f61414bb0..034f4949ae 100644 --- a/packages/orchestrator/internal/template/build/rootfs.go +++ b/packages/orchestrator/internal/template/build/rootfs.go @@ -176,6 +176,7 @@ ExecStart=-/sbin/agetty --noissue --autologin root %I 115200,38400,9600 vt102 ` hostname := "e2b.local" + eventProxyHostname := "event.e2b.com" hosts := fmt.Sprintf(`127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -184,7 +185,8 @@ ff00:: ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters 127.0.1.1 %s -`, hostname) +10.20.7.67 %s +`, hostname, eventProxyHostname) e2bFile := fmt.Sprintf(`ENV_ID=%s BUILD_ID=%s diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index e60b508ee8..94211f1a5a 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -39,9 +39,10 @@ type Closeable interface { } const ( - defaultPort = 5008 - defaultProxyPort = 5007 - defaultWait = 30 + defaultPort = 5008 + defaultProxyPort = 5007 + defaultEventProxyPort = 5010 + defaultWait = 30 version = "0.1.0" @@ -54,6 +55,7 @@ var commitSHA string func main() { port := flag.Uint("port", defaultPort, "orchestrator server port") proxyPort := flag.Uint("proxy-port", defaultProxyPort, "orchestrator proxy port") + eventProxyPort := flag.Uint("event-proxy-port", defaultEventProxyPort, "orchestrator event proxy port") wait := flag.Uint("wait", defaultWait, "orchestrator proxy port") flag.Parse() @@ -71,7 +73,7 @@ func main() { time.Sleep(time.Duration(*wait) * time.Second) } - success := run(*port, *proxyPort) + success := run(*port, *proxyPort, *eventProxyPort) log.Println("Stopping orchestrator, success:", success) @@ -80,7 +82,7 @@ func main() { } } -func run(port, proxyPort uint) (success bool) { +func run(port, proxyPort, eventProxyPort uint) (success bool) { success = true services := service.GetServices() @@ -220,6 +222,8 @@ func run(port, proxyPort uint) (success bool) { zap.L().Fatal("failed to create sandbox proxy", zap.Error(err)) } + eventProxy := proxy.NewEventProxy(eventProxyPort) + tracer := tel.TracerProvider.Tracer(serviceName) networkPool, err := network.NewPool(ctx, tel.MeterProvider, network.NewSlotsPoolSize, network.ReusedSlotsPoolSize, clientID, tracer) @@ -236,7 +240,7 @@ func run(port, proxyPort uint) (success bool) { grpcSrv := grpcserver.New(tel.TracerProvider, tel.MeterProvider, serviceInfo) - _, err = server.New(ctx, grpcSrv, tel, networkPool, devicePool, tracer, serviceInfo, sandboxProxy, sandboxes) + _, err = server.New(ctx, grpcSrv, tel, networkPool, devicePool, tracer, serviceInfo, sandboxProxy, eventProxy, sandboxes) if err != nil { zap.L().Fatal("failed to create server", zap.Error(err)) } @@ -310,6 +314,25 @@ func run(port, proxyPort uint) (success bool) { return nil }) + g.Go(func() error { + zap.L().Info("~~~Starting event proxy") + eventProxyErr := eventProxy.Start() + if eventProxyErr != nil && !errors.Is(eventProxyErr, http.ErrServerClosed) { + zap.L().Error("~~~error starting event proxy", zap.Error(eventProxyErr)) + + select { + case serviceError <- eventProxyErr: + default: + // Don't block if the serviceError channel is already closed + // or if the error is already sent + } + + return eventProxyErr + } + + return nil + }) + g.Go(func() (err error) { // this sets the error declared above so the function // in the defer can check it. From 587f92ed91030762421cc55cc7f950dd9f278634 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 16 Jun 2025 14:44:54 -0700 Subject: [PATCH 02/21] WIP working sandbox event endpoint boilerplate --- .../internal/proxy/event_proxy.go | 26 +++++-------------- .../internal/sandbox/network/network_linux.go | 4 +-- .../internal/template/build/rootfs.go | 4 +-- packages/orchestrator/main.go | 4 --- 4 files changed, 10 insertions(+), 28 deletions(-) diff --git a/packages/orchestrator/internal/proxy/event_proxy.go b/packages/orchestrator/internal/proxy/event_proxy.go index b1881a5188..df03a76280 100644 --- a/packages/orchestrator/internal/proxy/event_proxy.go +++ b/packages/orchestrator/internal/proxy/event_proxy.go @@ -3,9 +3,7 @@ package proxy import ( "context" "fmt" - "net" "net/http" - "time" "go.uber.org/zap" ) @@ -25,21 +23,9 @@ func NewEventProxy(port uint) *EventProxy { } } func (p *EventProxy) Start() error { - serverTransport := &http.Transport{ - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 8192, - IdleConnTimeout: 620 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 24 * time.Hour, - DisableKeepAlives: true, - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - zap.L().Info("Dialing", zap.String("network", network), zap.String("addr", addr)) - return net.Dial(network, addr) - }, - } - p.server.Handler = http.HandlerFunc(p.proxyHandler(serverTransport)) + p.server.Handler = http.HandlerFunc(p.proxyHandler()) - return p.server.ListenAndServeTLS("/etc/ssl/certs/cert.pem", "/etc/ssl/certs/key.pem") + return p.server.ListenAndServe() } func (p *EventProxy) Close(ctx context.Context) error { @@ -57,16 +43,16 @@ func (p *EventProxy) Close(ctx context.Context) error { return nil } -func (p *EventProxy) proxyHandler(transport *http.Transport) func(w http.ResponseWriter, r *http.Request) { +func (p *EventProxy) proxyHandler() func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { zap.L().Info("Forwarding request", zap.String("url", r.URL.String()), zap.String("method", r.Method)) - handleHTTP(w, r, transport) + handleHTTP(w, r) } } // handleHTTP handles regular HTTP requests -func handleHTTP(w http.ResponseWriter, r *http.Request, transport *http.Transport) { - zap.L().Info("[EVENT] handle HTTP request", zap.String("url", r.URL.String())) +func handleHTTP(w http.ResponseWriter, r *http.Request) { + zap.L().Info("[EVENT] handle event HTTP request", zap.String("url", r.URL.String())) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 0fed8fce0e..95a101d049 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -217,8 +217,8 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } - // Redirect HTTP traffic destined for 10.20.7.67 to local prox - err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", "10.20.7.67", "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") + // Redirect HTTP traffic destined for 8.8.8.7 to local prox + err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", "8.8.8.7", "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") if err != nil { return fmt.Errorf("error creating HTTP redirect rule to proxy: %w", err) } diff --git a/packages/orchestrator/internal/template/build/rootfs.go b/packages/orchestrator/internal/template/build/rootfs.go index 299fcc8214..f255a3a53c 100644 --- a/packages/orchestrator/internal/template/build/rootfs.go +++ b/packages/orchestrator/internal/template/build/rootfs.go @@ -181,7 +181,7 @@ ExecStart=-/sbin/agetty --noissue --autologin root %I 115200,38400,9600 vt102 ` hostname := "e2b.local" - eventProxyHostname := "event.e2b.com" + eventProxyHostname := "event.e2b.dev" hosts := fmt.Sprintf(`127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -190,7 +190,7 @@ ff00:: ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters 127.0.1.1 %s -10.20.7.67 %s +8.8.8.7 %s `, hostname, eventProxyHostname) e2bFile := fmt.Sprintf(`ENV_ID=%s diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 10ec20deeb..6d2ed58875 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -41,7 +41,6 @@ const ( defaultPort = 5008 defaultProxyPort = 5007 defaultEventProxyPort = 5010 - defaultWait = 30 version = "0.1.0" @@ -307,11 +306,8 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { }) g.Go(func() error { - zap.L().Info("~~~Starting event proxy") eventProxyErr := eventProxy.Start() if eventProxyErr != nil && !errors.Is(eventProxyErr, http.ErrServerClosed) { - zap.L().Error("~~~error starting event proxy", zap.Error(eventProxyErr)) - select { case serviceError <- eventProxyErr: default: From 61c8fe9b6058c53d4d4b6c91d4f71d9ef6f30a34 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 16 Jun 2025 18:33:10 -0700 Subject: [PATCH 03/21] renaming and changes in file structure --- .../internal/proxy/{event_proxy.go => event_server.go} | 0 packages/orchestrator/internal/sandbox/network/network_linux.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/orchestrator/internal/proxy/{event_proxy.go => event_server.go} (100%) diff --git a/packages/orchestrator/internal/proxy/event_proxy.go b/packages/orchestrator/internal/proxy/event_server.go similarity index 100% rename from packages/orchestrator/internal/proxy/event_proxy.go rename to packages/orchestrator/internal/proxy/event_server.go diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 95a101d049..590d92d2b1 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -217,7 +217,7 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } - // Redirect HTTP traffic destined for 8.8.8.7 to local prox + // Redirect http://event.e2b.dev traffic destined to event server err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", "8.8.8.7", "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") if err != nil { return fmt.Errorf("error creating HTTP redirect rule to proxy: %w", err) From b39dd561d65d2e58716bc86b29109a0132fc091b Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 19 Jun 2025 16:18:00 -0700 Subject: [PATCH 04/21] WIP abstract event server in its own sandbox submodule; add some header validation and register some event handlers --- .../internal/proxy/event_server.go | 64 ------------------ .../internal/sandbox/event/handlers.go | 37 ++++++++++ .../internal/sandbox/event/server.go | 67 +++++++++++++++++++ .../internal/sandbox/network/network_linux.go | 2 +- packages/orchestrator/internal/server/main.go | 3 +- packages/orchestrator/main.go | 7 +- 6 files changed, 111 insertions(+), 69 deletions(-) delete mode 100644 packages/orchestrator/internal/proxy/event_server.go create mode 100644 packages/orchestrator/internal/sandbox/event/handlers.go create mode 100644 packages/orchestrator/internal/sandbox/event/server.go diff --git a/packages/orchestrator/internal/proxy/event_server.go b/packages/orchestrator/internal/proxy/event_server.go deleted file mode 100644 index df03a76280..0000000000 --- a/packages/orchestrator/internal/proxy/event_server.go +++ /dev/null @@ -1,64 +0,0 @@ -package proxy - -import ( - "context" - "fmt" - "net/http" - - "go.uber.org/zap" -) - -// EventProxy handles outbound traffic from sandboxes calling the event.e2b.com domain -type EventProxy struct { - server *http.Server -} - -func NewEventProxy(port uint) *EventProxy { - server := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - } - - return &EventProxy{ - server: server, - } -} -func (p *EventProxy) Start() error { - p.server.Handler = http.HandlerFunc(p.proxyHandler()) - - return p.server.ListenAndServe() -} - -func (p *EventProxy) Close(ctx context.Context) error { - var err error - select { - case <-ctx.Done(): - err = p.server.Close() - default: - err = p.server.Shutdown(ctx) - } - if err != nil { - return fmt.Errorf("failed to shutdown proxy server: %w", err) - } - - return nil -} - -func (p *EventProxy) proxyHandler() func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - zap.L().Info("Forwarding request", zap.String("url", r.URL.String()), zap.String("method", r.Method)) - handleHTTP(w, r) - } -} - -// handleHTTP handles regular HTTP requests -func handleHTTP(w http.ResponseWriter, r *http.Request) { - zap.L().Info("[EVENT] handle event HTTP request", zap.String("url", r.URL.String())) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusCreated) - _, err := w.Write([]byte(`{"event_ack":true}`)) - if err != nil { - zap.L().Error("Failed to write response", zap.Error(err)) - } - return -} diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go new file mode 100644 index 0000000000..5d399daf7d --- /dev/null +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -0,0 +1,37 @@ +package event + +import "net/http" + +type EventHandler struct { + Path string + HandlerFunc func(w http.ResponseWriter, r *http.Request) +} + +var MetricsHandler = EventHandler{ + Path: "/metrics", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, err := w.Write([]byte(`{"event_ack":true,"path":"/metrics"}`)) + if err != nil { + http.Error(w, "Failed to write response", http.StatusInternalServerError) + return + } + }, +} + +var DefaultHandler = EventHandler{ + Path: "/", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Not found", http.StatusNotFound) + }, +} + +var EventHandlers = []EventHandler{ + MetricsHandler, + DefaultHandler, +} diff --git a/packages/orchestrator/internal/sandbox/event/server.go b/packages/orchestrator/internal/sandbox/event/server.go new file mode 100644 index 0000000000..af046b6685 --- /dev/null +++ b/packages/orchestrator/internal/sandbox/event/server.go @@ -0,0 +1,67 @@ +package event + +import ( + "context" + "fmt" + "net/http" +) + +// EventServer handles outbound HTTP requests from sandboxes calling the event.e2b.com endpoint +type EventServer struct { + server *http.Server +} + +func validateHeaders(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + sandboxID := r.Header.Get("E2B_SANDBOX_ID") + teamID := r.Header.Get("E2B_TEAM_ID") + + if sandboxID == "" || teamID == "" { + http.Error(w, "missing required headers", http.StatusBadRequest) + return + } + + // Add sandbox and team IDs to request context + ctx := context.WithValue(r.Context(), "sandboxID", sandboxID) + ctx = context.WithValue(ctx, "teamID", teamID) + r = r.WithContext(ctx) + + next.ServeHTTP(w, r) + } +} + +func NewEventServer(port uint, handlers []EventHandler) *EventServer { + mux := http.NewServeMux() + + for _, handler := range handlers { + mux.HandleFunc(handler.Path, validateHeaders(handler.HandlerFunc)) + } + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: mux, + } + + return &EventServer{ + server: server, + } +} + +func (p *EventServer) Start() error { + return p.server.ListenAndServe() +} + +func (p *EventServer) Close(ctx context.Context) error { + var err error + select { + case <-ctx.Done(): + err = p.server.Close() + default: + err = p.server.Shutdown(ctx) + } + if err != nil { + return fmt.Errorf("failed to shutdown event server: %w", err) + } + + return nil +} diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 590d92d2b1..126d9d0758 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -220,7 +220,7 @@ func (s *Slot) CreateNetwork() error { // Redirect http://event.e2b.dev traffic destined to event server err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", "8.8.8.7", "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") if err != nil { - return fmt.Errorf("error creating HTTP redirect rule to proxy: %w", err) + return fmt.Errorf("error creating HTTP redirect rule to sandbox event server: %w", err) } return nil diff --git a/packages/orchestrator/internal/server/main.go b/packages/orchestrator/internal/server/main.go index ddb8d3b19c..723ebe0e8f 100644 --- a/packages/orchestrator/internal/server/main.go +++ b/packages/orchestrator/internal/server/main.go @@ -12,6 +12,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/internal/grpcserver" "github.com/e2b-dev/infra/packages/orchestrator/internal/proxy" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/event" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/network" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/template" @@ -61,7 +62,7 @@ func New( tracer trace.Tracer, info *service.ServiceInfo, proxy *proxy.SandboxProxy, - eventProxy *proxy.EventProxy, + eventServer *event.EventServer, sandboxes *smap.Map[*sandbox.Sandbox], sandboxObserver *telemetry.SandboxObserver, featureFlags *featureflags.Client, diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 1207838847..13bb7884d0 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -21,6 +21,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/internal/grpcserver" "github.com/e2b-dev/infra/packages/orchestrator/internal/proxy" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/event" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/network" "github.com/e2b-dev/infra/packages/orchestrator/internal/server" @@ -220,7 +221,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { zap.L().Fatal("failed to create sandbox proxy", zap.Error(err)) } - eventProxy := proxy.NewEventProxy(eventProxyPort) + sbxEventServer := event.NewEventServer(eventProxyPort, event.EventHandlers) tracer := tel.TracerProvider.Tracer(serviceName) @@ -248,7 +249,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { zap.L().Fatal("failed to create sandbox observer", zap.Error(err)) } - _, err = server.New(ctx, grpcSrv, tel, networkPool, devicePool, tracer, serviceInfo, sandboxProxy, eventProxy, sandboxes, sandboxObserver, featureFlags) + _, err = server.New(ctx, grpcSrv, tel, networkPool, devicePool, tracer, serviceInfo, sandboxProxy, sbxEventServer, sandboxes, sandboxObserver, featureFlags) if err != nil { zap.L().Fatal("failed to create server", zap.Error(err)) } @@ -325,7 +326,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { }) g.Go(func() error { - eventProxyErr := eventProxy.Start() + eventProxyErr := sbxEventServer.Start() if eventProxyErr != nil && !errors.Is(eventProxyErr, http.ErrServerClosed) { select { case serviceError <- eventProxyErr: From 78bfc5a3355633947ae4bd3e275516959aceecd7 Mon Sep 17 00:00:00 2001 From: 0div Date: Fri, 20 Jun 2025 16:09:51 -0700 Subject: [PATCH 05/21] cosmetic edits --- packages/orchestrator/cmd/build-template/main.go | 2 +- .../orchestrator/internal/sandbox/event/handlers.go | 1 + packages/orchestrator/main.go | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/orchestrator/cmd/build-template/main.go b/packages/orchestrator/cmd/build-template/main.go index 339dc8221b..fd95600e5a 100644 --- a/packages/orchestrator/cmd/build-template/main.go +++ b/packages/orchestrator/cmd/build-template/main.go @@ -30,7 +30,7 @@ import ( ) const proxyPort = 5007 -const eventProxyPort = 5010 +const sbxEventServerPort = 5010 func main() { ctx, cancel := context.WithCancel(context.Background()) diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index 5d399daf7d..e0ca7539e0 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -7,6 +7,7 @@ type EventHandler struct { HandlerFunc func(w http.ResponseWriter, r *http.Request) } +// TODO: write an actual metrics handler var MetricsHandler = EventHandler{ Path: "/metrics", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 13bb7884d0..1b084ffd7c 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -80,7 +80,7 @@ func main() { } } -func run(port, proxyPort, eventProxyPort uint) (success bool) { +func run(port, proxyPort, sbxEventServerPort uint) (success bool) { success = true services := service.GetServices() @@ -221,7 +221,7 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { zap.L().Fatal("failed to create sandbox proxy", zap.Error(err)) } - sbxEventServer := event.NewEventServer(eventProxyPort, event.EventHandlers) + sbxEventServer := event.NewEventServer(sbxEventServerPort, event.EventHandlers) tracer := tel.TracerProvider.Tracer(serviceName) @@ -326,16 +326,16 @@ func run(port, proxyPort, eventProxyPort uint) (success bool) { }) g.Go(func() error { - eventProxyErr := sbxEventServer.Start() - if eventProxyErr != nil && !errors.Is(eventProxyErr, http.ErrServerClosed) { + sbxEventServerErr := sbxEventServer.Start() + if sbxEventServerErr != nil && !errors.Is(sbxEventServerErr, http.ErrServerClosed) { select { - case serviceError <- eventProxyErr: + case serviceError <- sbxEventServerErr: default: // Don't block if the serviceError channel is already closed // or if the error is already sent } - return eventProxyErr + return sbxEventServerErr } return nil From 6f1aa77588ab4648ff1d334486e4ac957ad56c72 Mon Sep 17 00:00:00 2001 From: 0div Date: Fri, 20 Jun 2025 16:46:13 -0700 Subject: [PATCH 06/21] address gofumpt errors --- packages/orchestrator/cmd/build-template/main.go | 6 ++++-- packages/orchestrator/internal/sandbox/event/handlers.go | 2 +- packages/orchestrator/internal/sandbox/event/server.go | 5 ----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/orchestrator/cmd/build-template/main.go b/packages/orchestrator/cmd/build-template/main.go index fd95600e5a..b1fb552967 100644 --- a/packages/orchestrator/cmd/build-template/main.go +++ b/packages/orchestrator/cmd/build-template/main.go @@ -29,8 +29,10 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/storage" ) -const proxyPort = 5007 -const sbxEventServerPort = 5010 +const ( + proxyPort = 5007 + sbxEventServerPort = 5010 +) func main() { ctx, cancel := context.WithCancel(context.Background()) diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index e0ca7539e0..e76066583b 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -7,10 +7,10 @@ type EventHandler struct { HandlerFunc func(w http.ResponseWriter, r *http.Request) } -// TODO: write an actual metrics handler var MetricsHandler = EventHandler{ Path: "/metrics", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + // TODO: write an actual metrics handler if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return diff --git a/packages/orchestrator/internal/sandbox/event/server.go b/packages/orchestrator/internal/sandbox/event/server.go index af046b6685..dab6417720 100644 --- a/packages/orchestrator/internal/sandbox/event/server.go +++ b/packages/orchestrator/internal/sandbox/event/server.go @@ -21,11 +21,6 @@ func validateHeaders(next http.HandlerFunc) http.HandlerFunc { return } - // Add sandbox and team IDs to request context - ctx := context.WithValue(r.Context(), "sandboxID", sandboxID) - ctx = context.WithValue(ctx, "teamID", teamID) - r = r.WithContext(ctx) - next.ServeHTTP(w, r) } } From 087c42145534b3262575f13482d07867daf417e6 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 23 Jun 2025 15:26:51 -0700 Subject: [PATCH 07/21] store events in redis via a catchall endpoint --- packages/nomad/main.tf | 3 +- packages/nomad/orchestrator.hcl | 1 + packages/orchestrator/go.mod | 2 + packages/orchestrator/go.sum | 4 + .../internal/sandbox/event/handlers.go | 118 ++++++++++++++---- .../internal/sandbox/event/server.go | 5 +- packages/orchestrator/main.go | 20 ++- 7 files changed, 124 insertions(+), 29 deletions(-) diff --git a/packages/nomad/main.tf b/packages/nomad/main.tf index 2dd77f6774..c92d6f92c6 100644 --- a/packages/nomad/main.tf +++ b/packages/nomad/main.tf @@ -402,6 +402,7 @@ locals { otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}" allow_sandbox_internet = var.allow_sandbox_internet launch_darkly_api_key = trimspace(data.google_secret_manager_secret_version.launch_darkly_api_key.secret_data) + redis_url = data.google_secret_manager_secret_version.redis_url.secret_data != "redis.service.consul" ? "${data.google_secret_manager_secret_version.redis_url.secret_data}:${var.redis_port.port}" : "redis.service.consul:${var.redis_port.port}" } orchestrator_job_check = templatefile("${path.module}/orchestrator.hcl", merge( @@ -651,4 +652,4 @@ resource "nomad_job" "clickhouse_migrator" { clickhouse_password = random_password.clickhouse_password.result clickhouse_port = var.clickhouse_server_port.port }) -} \ No newline at end of file +} diff --git a/packages/nomad/orchestrator.hcl b/packages/nomad/orchestrator.hcl index d35b799378..a1b18fa88e 100644 --- a/packages/nomad/orchestrator.hcl +++ b/packages/nomad/orchestrator.hcl @@ -70,6 +70,7 @@ EOT TEMPLATE_BUCKET_NAME = "${template_bucket_name}" OTEL_COLLECTOR_GRPC_ENDPOINT = "${otel_collector_grpc_endpoint}" ALLOW_SANDBOX_INTERNET = "${allow_sandbox_internet}" + REDIS_URL = "${redis_url}" %{ if launch_darkly_api_key != "" } LAUNCH_DARKLY_API_KEY = "${launch_darkly_api_key}" diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index cc149a5d29..3611d39dea 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -91,6 +91,7 @@ require ( github.com/containernetworking/cni v1.2.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/uniuri v1.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/cli v28.1.1+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect @@ -169,6 +170,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/redis/go-redis/v9 v9.10.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/ulikunitz/xz v0.5.12 // indirect diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 94874953aa..79020b0920 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -335,6 +335,8 @@ github.com/dchest/uniuri v1.2.0/go.mod h1:fSzm4SLHzNZvWLvWJew423PhAzkpNQYq+uNLq4 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= @@ -937,6 +939,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs= +github.com/redis/go-redis/v9 v9.10.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index e76066583b..e36e456271 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -1,38 +1,104 @@ package event -import "net/http" +import ( + "encoding/json" + "io" + "net/http" -type EventHandler struct { - Path string - HandlerFunc func(w http.ResponseWriter, r *http.Request) + "github.com/redis/go-redis/v9" +) + +type EventHandler interface { + Path() string + HandlerFunc(w http.ResponseWriter, r *http.Request) } -var MetricsHandler = EventHandler{ - Path: "/metrics", - HandlerFunc: func(w http.ResponseWriter, r *http.Request) { - // TODO: write an actual metrics handler - if r.Method != http.MethodPost { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusCreated) - _, err := w.Write([]byte(`{"event_ack":true,"path":"/metrics"}`)) +type EventData struct { + Path string `json:"path"` + Body map[string]any `json:"body"` +} + +type MetricsHandler struct{} + +func (h *MetricsHandler) Path() string { + return "/metrics" +} + +func (h *MetricsHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, err := w.Write([]byte(`{"event_ack":true,"path":"/metrics"}`)) + if err != nil { + http.Error(w, "Failed to write response", http.StatusInternalServerError) + return + } +} + +// This handler is used to store event data for all paths that are not registered in the event server. +// This is used to track ad-hoc events that are not handled by the event server. +type DefaultHandler struct { + redisClient redis.UniversalClient +} + +func (h *DefaultHandler) Path() string { + return "/" +} + +func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { + sandboxID := r.Header.Get("E2B_SANDBOX_ID") + + if r.Method == http.MethodGet { + body, err := h.redisClient.Get(r.Context(), sandboxID).Result() if err != nil { - http.Error(w, "Failed to write response", http.StatusInternalServerError) + http.Error(w, "Failed to get event data for sandbox "+sandboxID, http.StatusInternalServerError) return } - }, -} + w.WriteHeader(http.StatusOK) + w.Write([]byte(body)) + return + } + + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Create event data with path and body + eventData := EventData{ + Path: r.URL.Path, + } + + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusInternalServerError) + return + } + + eventData.Body = make(map[string]any) + err = json.Unmarshal(body, &eventData.Body) + if err != nil { + http.Error(w, "Failed to unmarshal request body", http.StatusInternalServerError) + return + } + + // Store in Redis with sandboxID as key + err = h.redisClient.Set(r.Context(), sandboxID, eventData, 0).Err() + if err != nil { + http.Error(w, "Failed to store event data", http.StatusInternalServerError) + return + } -var DefaultHandler = EventHandler{ - Path: "/", - HandlerFunc: func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "Not found", http.StatusNotFound) - }, + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"event_ack":true}`)) } -var EventHandlers = []EventHandler{ - MetricsHandler, - DefaultHandler, +func NewEventHandlers(redisClient redis.UniversalClient) []EventHandler { + return []EventHandler{ + &MetricsHandler{}, + &DefaultHandler{redisClient}, + } } diff --git a/packages/orchestrator/internal/sandbox/event/server.go b/packages/orchestrator/internal/sandbox/event/server.go index dab6417720..d7e407ab01 100644 --- a/packages/orchestrator/internal/sandbox/event/server.go +++ b/packages/orchestrator/internal/sandbox/event/server.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/http" + + "go.uber.org/zap" ) // EventServer handles outbound HTTP requests from sandboxes calling the event.e2b.com endpoint @@ -29,7 +31,7 @@ func NewEventServer(port uint, handlers []EventHandler) *EventServer { mux := http.NewServeMux() for _, handler := range handlers { - mux.HandleFunc(handler.Path, validateHeaders(handler.HandlerFunc)) + mux.HandleFunc(handler.Path(), validateHeaders(handler.HandlerFunc)) } server := &http.Server{ @@ -43,6 +45,7 @@ func NewEventServer(port uint, handlers []EventHandler) *EventServer { } func (p *EventServer) Start() error { + zap.L().Info("Starting event server") return p.server.ListenAndServe() } diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index 1b084ffd7c..b9c8d18de5 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -34,6 +34,7 @@ import ( sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "github.com/redis/go-redis/v9" ) type Closeable interface { @@ -221,7 +222,24 @@ func run(port, proxyPort, sbxEventServerPort uint) (success bool) { zap.L().Fatal("failed to create sandbox proxy", zap.Error(err)) } - sbxEventServer := event.NewEventServer(sbxEventServerPort, event.EventHandlers) + var redisClient redis.UniversalClient + if redisClusterUrl := os.Getenv("REDIS_CLUSTER_URL"); redisClusterUrl != "" { + redisClient = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{redisClusterUrl}, + MinIdleConns: 1, + }) + } else if redisUrl := os.Getenv("REDIS_URL"); redisUrl != "" { + redisClient = redis.NewClient(&redis.Options{ + Addr: redisUrl, + MinIdleConns: 1, + }) + } else { + zap.L().Fatal("REDIS_URL not set") + } + + sbxEventHandlers := event.NewEventHandlers(redisClient) + + sbxEventServer := event.NewEventServer(sbxEventServerPort, sbxEventHandlers) tracer := tel.TracerProvider.Tracer(serviceName) From 19a18a7ff81f26e41f8c3f62dcb11dc8bd08a7ee Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 24 Jun 2025 14:18:22 -0700 Subject: [PATCH 08/21] create event store interface --- .../orchestrator/cmd/build-template/main.go | 3 +- packages/orchestrator/go.mod | 2 +- packages/orchestrator/go.sum | 4 + .../internal/sandbox/event/handlers.go | 25 +++--- .../internal/sandbox/event/store.go | 82 +++++++++++++++++++ packages/orchestrator/main.go | 10 ++- 6 files changed, 107 insertions(+), 19 deletions(-) create mode 100644 packages/orchestrator/internal/sandbox/event/store.go diff --git a/packages/orchestrator/cmd/build-template/main.go b/packages/orchestrator/cmd/build-template/main.go index b1fb552967..bd2b10363e 100644 --- a/packages/orchestrator/cmd/build-template/main.go +++ b/packages/orchestrator/cmd/build-template/main.go @@ -30,8 +30,7 @@ import ( ) const ( - proxyPort = 5007 - sbxEventServerPort = 5010 + proxyPort = 5007 ) func main() { diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index 3611d39dea..f29877e584 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -25,6 +25,7 @@ require ( github.com/ngrok/firewall_toolkit v0.0.18 github.com/pkg/errors v0.9.1 github.com/pojntfx/go-nbd v0.3.2 + github.com/redis/go-redis/v9 v9.10.0 github.com/rs/zerolog v1.34.0 github.com/soheilhy/cmux v0.1.5 github.com/stretchr/testify v1.10.0 @@ -170,7 +171,6 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/redis/go-redis/v9 v9.10.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/ulikunitz/xz v0.5.12 // indirect diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index 79020b0920..9cf3f5a2d7 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -174,6 +174,10 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index e36e456271..4ab8b4b364 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -1,11 +1,12 @@ package event import ( + "context" "encoding/json" "io" "net/http" - "github.com/redis/go-redis/v9" + "go.uber.org/zap" ) type EventHandler interface { @@ -13,11 +14,6 @@ type EventHandler interface { HandlerFunc(w http.ResponseWriter, r *http.Request) } -type EventData struct { - Path string `json:"path"` - Body map[string]any `json:"body"` -} - type MetricsHandler struct{} func (h *MetricsHandler) Path() string { @@ -41,7 +37,7 @@ func (h *MetricsHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { // This handler is used to store event data for all paths that are not registered in the event server. // This is used to track ad-hoc events that are not handled by the event server. type DefaultHandler struct { - redisClient redis.UniversalClient + store SandboxEventStore } func (h *DefaultHandler) Path() string { @@ -50,15 +46,17 @@ func (h *DefaultHandler) Path() string { func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { sandboxID := r.Header.Get("E2B_SANDBOX_ID") + zap.L().Info("~~~[DefaultHandler] Received event", zap.String("method", r.Method), zap.String("path", r.URL.Path), zap.String("sandboxID", sandboxID)) if r.Method == http.MethodGet { - body, err := h.redisClient.Get(r.Context(), sandboxID).Result() + body, err := h.store.GetSandbox(sandboxID) if err != nil { + zap.L().Error("Failed to get event data for sandbox "+sandboxID, zap.Error(err)) http.Error(w, "Failed to get event data for sandbox "+sandboxID, http.StatusInternalServerError) return } w.WriteHeader(http.StatusOK) - w.Write([]byte(body)) + w.Write([]byte(body.Path)) return } @@ -68,7 +66,7 @@ func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { } // Create event data with path and body - eventData := EventData{ + eventData := SandboxEvent{ Path: r.URL.Path, } @@ -86,8 +84,9 @@ func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { } // Store in Redis with sandboxID as key - err = h.redisClient.Set(r.Context(), sandboxID, eventData, 0).Err() + err = h.store.StoreSandbox(sandboxID, &eventData, 0) if err != nil { + zap.L().Error("Failed to store event data", zap.Error(err)) http.Error(w, "Failed to store event data", http.StatusInternalServerError) return } @@ -96,9 +95,9 @@ func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"event_ack":true}`)) } -func NewEventHandlers(redisClient redis.UniversalClient) []EventHandler { +func NewEventHandlers(ctx context.Context, store SandboxEventStore) []EventHandler { return []EventHandler{ &MetricsHandler{}, - &DefaultHandler{redisClient}, + &DefaultHandler{store}, } } diff --git a/packages/orchestrator/internal/sandbox/event/store.go b/packages/orchestrator/internal/sandbox/event/store.go new file mode 100644 index 0000000000..9314a0ee6f --- /dev/null +++ b/packages/orchestrator/internal/sandbox/event/store.go @@ -0,0 +1,82 @@ +package event + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/trace" +) + +type SandboxEvent struct { + Path string `json:"path"` + Body map[string]any `json:"body"` +} + +func (i SandboxEvent) MarshalBinary() ([]byte, error) { + return json.Marshal(i) +} + +type sandboxEventStore struct { + ctx context.Context + tracer trace.Tracer + redisClient redis.UniversalClient +} + +type SandboxEventStore interface { + GetSandbox(sandboxId string) (*SandboxEvent, error) + StoreSandbox(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error + DeleteSandbox(sandboxId string) error + Close() error +} + +func NewMemorySandboxesEvent(ctx context.Context, tracer trace.Tracer, redisClient redis.UniversalClient) SandboxEventStore { + return &sandboxEventStore{ + ctx: ctx, + tracer: tracer, + redisClient: redisClient, + } +} + +func (c *sandboxEventStore) GetSandbox(sandboxId string) (*SandboxEvent, error) { + _, span := c.tracer.Start(c.ctx, "sandbox-event-get") + defer span.End() + + body, err := c.redisClient.Get(c.ctx, sandboxId).Result() + if err != nil { + return nil, err + } + + return &SandboxEvent{ + Path: string(body), + Body: make(map[string]any), + }, nil +} + +func (c *sandboxEventStore) StoreSandbox(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error { + _, span := c.tracer.Start(c.ctx, "sandbox-event-store") + defer span.End() + + body, err := c.redisClient.Get(c.ctx, sandboxId).Result() + if err != nil { + if err == redis.Nil { + return fmt.Errorf("sandbox event not found: %w", err) + } + return err + } + + return c.redisClient.Set(c.ctx, sandboxId, body, expiration).Err() +} + +func (c *sandboxEventStore) DeleteSandbox(sandboxId string) error { + _, span := c.tracer.Start(c.ctx, "sandbox-event-delete") + defer span.End() + + return c.redisClient.Del(c.ctx, sandboxId).Err() +} + +func (c *sandboxEventStore) Close() error { + return c.redisClient.Close() +} diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index b9c8d18de5..a0d6c281a8 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -223,6 +223,8 @@ func run(port, proxyPort, sbxEventServerPort uint) (success bool) { } var redisClient redis.UniversalClient + defer redisClient.Close() + if redisClusterUrl := os.Getenv("REDIS_CLUSTER_URL"); redisClusterUrl != "" { redisClient = redis.NewClusterClient(&redis.ClusterOptions{ Addrs: []string{redisClusterUrl}, @@ -237,11 +239,13 @@ func run(port, proxyPort, sbxEventServerPort uint) (success bool) { zap.L().Fatal("REDIS_URL not set") } - sbxEventHandlers := event.NewEventHandlers(redisClient) + tracer := tel.TracerProvider.Tracer(serviceName) - sbxEventServer := event.NewEventServer(sbxEventServerPort, sbxEventHandlers) + eventStore := event.NewMemorySandboxesEvent(ctx, tracer, redisClient) + defer eventStore.Close() + sbxEventHandlers := event.NewEventHandlers(ctx, eventStore) - tracer := tel.TracerProvider.Tracer(serviceName) + sbxEventServer := event.NewEventServer(sbxEventServerPort, sbxEventHandlers) networkPool, err := network.NewPool(ctx, tel.MeterProvider, network.NewSlotsPoolSize, network.ReusedSlotsPoolSize, clientID, tracer) if err != nil { From 5aa2b58b893883ed27383331c99b167e06188eaa Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 24 Jun 2025 16:55:34 -0700 Subject: [PATCH 09/21] go mod tidy & go work sync --- packages/api/go.mod | 2 +- packages/api/go.sum | 3 +-- packages/client-proxy/go.mod | 2 +- packages/client-proxy/go.sum | 3 +-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/api/go.mod b/packages/api/go.mod index b0b7157671..a211437ff9 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -37,7 +37,7 @@ require ( github.com/jackc/pgx/v5 v5.7.4 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.24.2 - github.com/redis/go-redis/v9 v9.8.0 + github.com/redis/go-redis/v9 v9.10.0 github.com/stretchr/testify v1.10.0 google.golang.org/grpc v1.72.1 ) diff --git a/packages/api/go.sum b/packages/api/go.sum index 8e51b3ec50..5e820b2cf7 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -843,8 +843,7 @@ github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeM github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510 h1:6ksZ7t1hNOzGPPs8DK7SvXQf6UfWzi+W5Z7PCBl8gx4= github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510/go.mod h1:UC0TwJiF90m2T3iYPQBKnGu8gv3s55dF/EgpTq8gyvo= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= -github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= -github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= diff --git a/packages/client-proxy/go.mod b/packages/client-proxy/go.mod index 3decb6dcda..a4da37b00c 100644 --- a/packages/client-proxy/go.mod +++ b/packages/client-proxy/go.mod @@ -16,7 +16,7 @@ require ( github.com/jellydator/ttlcache/v3 v3.3.1-0.20250207140243-aefc35918359 github.com/miekg/dns v1.1.63 github.com/oapi-codegen/gin-middleware v1.0.2 - github.com/redis/go-redis/v9 v9.8.0 + github.com/redis/go-redis/v9 v9.10.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 go.opentelemetry.io/otel/metric v1.36.0 go.opentelemetry.io/otel/trace v1.36.0 diff --git a/packages/client-proxy/go.sum b/packages/client-proxy/go.sum index 8d50e5ac16..c8f0f6b023 100644 --- a/packages/client-proxy/go.sum +++ b/packages/client-proxy/go.sum @@ -669,8 +669,7 @@ github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2b github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510 h1:6ksZ7t1hNOzGPPs8DK7SvXQf6UfWzi+W5Z7PCBl8gx4= github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510/go.mod h1:UC0TwJiF90m2T3iYPQBKnGu8gv3s55dF/EgpTq8gyvo= -github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= -github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= From be5c2ade7e283bd9a4fe1fbccb6a33a63fda0bde Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 24 Jun 2025 17:07:26 -0700 Subject: [PATCH 10/21] go mod tidy api --- packages/api/go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/go.sum b/packages/api/go.sum index 5e820b2cf7..5886f00e11 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -844,6 +844,7 @@ github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510 h1:6ksZ7t github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510/go.mod h1:UC0TwJiF90m2T3iYPQBKnGu8gv3s55dF/EgpTq8gyvo= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.10.0 h1:FxwK3eV8p/CQa0Ch276C7u2d0eNC9kCmAYQ7mCXCzVs= +github.com/redis/go-redis/v9 v9.10.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= From eb5ca83da4cd8346886448156b3df806cccbfc22 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 24 Jun 2025 18:00:11 -0700 Subject: [PATCH 11/21] use sorted set for events time-series --- .../internal/sandbox/event/handlers.go | 6 +-- .../internal/sandbox/event/store.go | 48 ++++++++++++++++--- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index cfbc349787..f5e6979b68 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -47,14 +47,14 @@ func (h *DefaultHandler) Path() string { func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { sandboxID := r.Header.Get("E2B_SANDBOX_ID") if r.Method == http.MethodGet { - event, err := h.store.GetEvent(sandboxID) + events, err := h.store.GetLastNEvents(sandboxID, 10) if err != nil { zap.L().Error("Failed to get event data for sandbox "+sandboxID, zap.Error(err)) http.Error(w, "Failed to get event data for sandbox "+sandboxID, http.StatusInternalServerError) return } - eventJSON, err := json.Marshal(event) + eventJSON, err := json.Marshal(events) if err != nil { zap.L().Error("Failed to marshal event data", zap.Error(err)) http.Error(w, "Failed to marshal event data", http.StatusInternalServerError) @@ -91,7 +91,7 @@ func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { } // Store in Redis with sandboxID as key - err = h.store.SetEvent(sandboxID, &eventData, 0) + err = h.store.AddEvent(sandboxID, &eventData, 0) if err != nil { zap.L().Error("Failed to store event data", zap.Error(err)) http.Error(w, "Failed to store event data", http.StatusInternalServerError) diff --git a/packages/orchestrator/internal/sandbox/event/store.go b/packages/orchestrator/internal/sandbox/event/store.go index 2b6287b051..600569e62b 100644 --- a/packages/orchestrator/internal/sandbox/event/store.go +++ b/packages/orchestrator/internal/sandbox/event/store.go @@ -25,8 +25,9 @@ type sandboxEventStore struct { } type SandboxEventStore interface { - GetEvent(sandboxId string) (*SandboxEvent, error) - SetEvent(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error + GetLastEvent(sandboxId string) (*SandboxEvent, error) + GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) + AddEvent(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error DelEvent(sandboxId string) error Close() error } @@ -39,14 +40,18 @@ func NewSandboxEventStore(ctx context.Context, tracer trace.Tracer, redisClient } } -func (c *sandboxEventStore) GetEvent(sandboxId string) (*SandboxEvent, error) { - _, span := c.tracer.Start(c.ctx, "sandbox-event-get") +func (c *sandboxEventStore) GetLastEvent(sandboxId string) (*SandboxEvent, error) { + _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last") defer span.End() - rawEvent, err := c.redisClient.Get(c.ctx, sandboxId).Result() + result, err := c.redisClient.ZRevRangeWithScores(c.ctx, sandboxId, 0, 0).Result() if err != nil { return nil, err } + if len(result) == 0 { + return nil, redis.Nil + } + rawEvent := result[0].Member.(string) var event SandboxEvent err = json.Unmarshal([]byte(rawEvent), &event) @@ -56,11 +61,40 @@ func (c *sandboxEventStore) GetEvent(sandboxId string) (*SandboxEvent, error) { return &event, nil } -func (c *sandboxEventStore) SetEvent(sandboxId string, event *SandboxEvent, expiration time.Duration) error { +func (c *sandboxEventStore) GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) { + _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last-n") + defer span.End() + + result, err := c.redisClient.ZRevRangeWithScores(c.ctx, sandboxId, 0, int64(n-1)).Result() + if err != nil { + return nil, err + } + if len(result) == 0 { + return nil, redis.Nil + } + + events := make([]*SandboxEvent, 0, len(result)) + for _, item := range result { + rawEvent := item.Member.(string) + var event SandboxEvent + err = json.Unmarshal([]byte(rawEvent), &event) + if err != nil { + return nil, err + } + events = append(events, &event) + } + + return events, nil +} + +func (c *sandboxEventStore) AddEvent(sandboxId string, event *SandboxEvent, expiration time.Duration) error { _, span := c.tracer.Start(c.ctx, "sandbox-event-store") defer span.End() - return c.redisClient.Set(c.ctx, sandboxId, event, expiration).Err() + return c.redisClient.ZAdd(c.ctx, sandboxId, redis.Z{ + Score: float64(time.Now().UnixNano()), + Member: event, + }).Err() } func (c *sandboxEventStore) DelEvent(sandboxId string) error { From 31797896f683ccc352ba26cf9559f262c57241a3 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 25 Jun 2025 10:35:30 -0700 Subject: [PATCH 12/21] go-lint problematic file --- packages/orchestrator/internal/sandbox/event/handlers.go | 3 +++ packages/orchestrator/main.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index f5e6979b68..0a0797c5fb 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -83,9 +83,12 @@ func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { return } + zap.L().Info("Received event", zap.String("body", string(body))) + eventData.Body = make(map[string]any) err = json.Unmarshal(body, &eventData.Body) if err != nil { + zap.L().Error("Failed to unmarshal request body", zap.Error(err)) http.Error(w, "Failed to unmarshal request body", http.StatusInternalServerError) return } diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index e12974ac0b..c35bb04457 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/redis/go-redis/v9" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" @@ -35,7 +36,6 @@ import ( sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox" "github.com/e2b-dev/infra/packages/shared/pkg/smap" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" - "github.com/redis/go-redis/v9" ) type Closeable interface { From 30e2cd8d3ca7a990e61f0f10548a00bc15853b69 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 25 Jun 2025 11:08:02 -0700 Subject: [PATCH 13/21] add redis to start-services action --- .github/actions/start-services/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 9df0d6c156..b8fdfbf080 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -44,6 +44,11 @@ runs: make -C tests/integration seed shell: bash + - name: Start Redis + run: | + docker run -d --name redis -p 6379:6379 redis:7.4.2-alpine + shell: bash + - name: Start Services env: ENVD_TIMEOUT: "60s" @@ -51,6 +56,7 @@ runs: SANDBOX_ACCESS_TOKEN_HASH_SEED: "abcdefghijklmnopqrstuvwxyz" SUPABASE_JWT_SECRETS: "supabasejwtsecretsupabasejwtsecret" TEMPLATE_MANAGER_ADDRESS: "http://localhost:5008" + REDIS_URL: "redis://localhost:6379" ARTIFACTS_REGISTRY_PROVIDER: "Local" STORAGE_PROVIDER: "Local" ENVIRONMENT: "local" From 881fbb0de2af7f98cb1e2d043ba1f0c72a4c7da9 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 25 Jun 2025 11:20:13 -0700 Subject: [PATCH 14/21] add redis_url to template-manager iac --- packages/nomad/main.tf | 1 + packages/nomad/template-manager.hcl | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/nomad/main.tf b/packages/nomad/main.tf index d038f904a9..f027a23848 100644 --- a/packages/nomad/main.tf +++ b/packages/nomad/main.tf @@ -485,6 +485,7 @@ resource "nomad_job" "template_manager" { logs_collector_public_ip = var.logs_proxy_address orchestrator_services = "template-manager" allow_sandbox_internet = var.allow_sandbox_internet + redis_url = data.google_secret_manager_secret_version.redis_url.secret_data != "redis.service.consul" ? "${data.google_secret_manager_secret_version.redis_url.secret_data}:${var.redis_port.port}" : "redis.service.consul:${var.redis_port.port}" }) } resource "nomad_job" "loki" { diff --git a/packages/nomad/template-manager.hcl b/packages/nomad/template-manager.hcl index 6987ed5916..a3e0f1c589 100644 --- a/packages/nomad/template-manager.hcl +++ b/packages/nomad/template-manager.hcl @@ -64,6 +64,7 @@ job "template-manager" { ORCHESTRATOR_SERVICES = "${orchestrator_services}" LOGS_COLLECTOR_PUBLIC_IP = "${logs_collector_public_ip}" ALLOW_SANDBOX_INTERNET = "${allow_sandbox_internet}" + REDIS_URL = "${redis_url}" %{ if !update_stanza } FORCE_STOP = "true" %{ endif } From 89eb6fbdf608353fde953b8eae55be9801aeb006 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 25 Jun 2025 11:40:22 -0700 Subject: [PATCH 15/21] remove proto from redis url in github actions --- .github/actions/start-services/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index b8fdfbf080..a156511f31 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -56,7 +56,7 @@ runs: SANDBOX_ACCESS_TOKEN_HASH_SEED: "abcdefghijklmnopqrstuvwxyz" SUPABASE_JWT_SECRETS: "supabasejwtsecretsupabasejwtsecret" TEMPLATE_MANAGER_ADDRESS: "http://localhost:5008" - REDIS_URL: "redis://localhost:6379" + REDIS_URL: "localhost:6379" ARTIFACTS_REGISTRY_PROVIDER: "Local" STORAGE_PROVIDER: "Local" ENVIRONMENT: "local" From fdc5cdb6bf22fbbc4732ddd5d37d1bdbf0e1ca8d Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 25 Jun 2025 18:10:25 -0700 Subject: [PATCH 16/21] update openapi spec for sandbox events endpoint --- spec/openapi.yml | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/spec/openapi.yml b/spec/openapi.yml index ccad921984..19360bc545 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -224,6 +224,22 @@ components: type: integer format: int64 description: Total memory in MiB + + SandboxEvent: + required: + - timestamp + - path + properties: + timestamp: + type: integer + format: int64 + description: Unix timestamp of the event in UTC + path: + type: string + description: Path of the event + body: + type: object + description: Body of the event Sandbox: required: @@ -1148,6 +1164,49 @@ paths: "500": $ref: "#/components/responses/500" + /sandboxes/{sandboxID}/events: + get: + description: Get sandbox events + tags: [sandboxes] + security: + - ApiKeyAuth: [] + - Supabase1TokenAuth: [] + Supabase2TeamAuth: [] + parameters: + - $ref: "#/components/parameters/sandboxID" + - in: query + name: offset + schema: + type: integer + format: int32 + minimum: 0 + default: 0 + description: Number of events to skip + - in: query + name: limit + schema: + type: integer + format: int32 + minimum: 1 + default: 100 + maximum: 1000 + description: Maximum number of events to return + responses: + "200": + description: Sandbox events + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/SandboxEvent" + "401": + $ref: "#/components/responses/401" + "404": + $ref: "#/components/responses/404" + "500": + $ref: "#/components/responses/500" + /sandboxes/{sandboxID}/refreshes: post: description: Refresh the sandbox extending its time to live From b4bb3f6bdcf12b08b6ba51de19d3f5930d3f4bf6 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 8 Jul 2025 16:25:09 -0700 Subject: [PATCH 17/21] track Sandbox IPs via store during key Sandbox lifecycle steps relating to network slots and use it to map back to sandbox ID in event handler --- .../internal/sandbox/event/handlers.go | 14 ++++++-- .../internal/sandbox/event/store.go | 32 ++++++++++++++++--- .../orchestrator/internal/sandbox/sandbox.go | 8 +++++ packages/orchestrator/internal/server/main.go | 4 +++ .../orchestrator/internal/server/sandboxes.go | 1 + packages/orchestrator/main.go | 3 +- 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/event/handlers.go b/packages/orchestrator/internal/sandbox/event/handlers.go index 0a0797c5fb..b3cadcbcd6 100644 --- a/packages/orchestrator/internal/sandbox/event/handlers.go +++ b/packages/orchestrator/internal/sandbox/event/handlers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "net/http" + "strings" "go.uber.org/zap" ) @@ -34,7 +35,6 @@ func (h *MetricsHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { } } -// This handler is used to store event data for all paths that are not registered in the event server. // This is used to track ad-hoc events that are not handled by the event server. type DefaultHandler struct { store SandboxEventStore @@ -45,7 +45,17 @@ func (h *DefaultHandler) Path() string { } func (h *DefaultHandler) HandlerFunc(w http.ResponseWriter, r *http.Request) { - sandboxID := r.Header.Get("E2B_SANDBOX_ID") + addr := r.RemoteAddr + ip := strings.Split(addr, ":")[0] + sandboxID, err := h.store.GetSandboxIP(ip) + if err != nil { + zap.L().Error("Failed to get sandbox ID from IP", zap.Error(err)) + http.Error(w, "Error handling event", http.StatusInternalServerError) + return + } + + zap.L().Debug("Received request from sandbox", zap.String("sandbox_id", sandboxID), zap.String("ip", ip)) + if r.Method == http.MethodGet { events, err := h.store.GetLastNEvents(sandboxID, 10) if err != nil { diff --git a/packages/orchestrator/internal/sandbox/event/store.go b/packages/orchestrator/internal/sandbox/event/store.go index 600569e62b..3bcbe4ad9c 100644 --- a/packages/orchestrator/internal/sandbox/event/store.go +++ b/packages/orchestrator/internal/sandbox/event/store.go @@ -9,6 +9,13 @@ import ( "go.opentelemetry.io/otel/trace" ) +const ( + cacheTL = time.Hour * 24 * 30 + + EventPrefix = "ev:" + IPPrefix = "ip:" +) + type SandboxEvent struct { Path string `json:"path"` Body map[string]any `json:"body"` @@ -25,10 +32,15 @@ type sandboxEventStore struct { } type SandboxEventStore interface { + SetSandboxIP(sandboxId string, ip string) error + GetSandboxIP(sandboxId string) (string, error) + DelSandboxIP(sandboxId string) error + GetLastEvent(sandboxId string) (*SandboxEvent, error) GetLastNEvents(sandboxId string, n int) ([]*SandboxEvent, error) AddEvent(sandboxId string, SandboxEvent *SandboxEvent, expiration time.Duration) error DelEvent(sandboxId string) error + Close() error } @@ -40,11 +52,23 @@ func NewSandboxEventStore(ctx context.Context, tracer trace.Tracer, redisClient } } +func (c *sandboxEventStore) SetSandboxIP(sandboxId string, ip string) error { + return c.redisClient.Set(c.ctx, IPPrefix+ip, sandboxId, cacheTL).Err() +} + +func (c *sandboxEventStore) GetSandboxIP(ip string) (string, error) { + return c.redisClient.Get(c.ctx, IPPrefix+ip).Result() +} + +func (c *sandboxEventStore) DelSandboxIP(ip string) error { + return c.redisClient.Del(c.ctx, IPPrefix+ip).Err() +} + func (c *sandboxEventStore) GetLastEvent(sandboxId string) (*SandboxEvent, error) { _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last") defer span.End() - result, err := c.redisClient.ZRevRangeWithScores(c.ctx, sandboxId, 0, 0).Result() + result, err := c.redisClient.ZRevRangeWithScores(c.ctx, EventPrefix+sandboxId, 0, 0).Result() if err != nil { return nil, err } @@ -65,7 +89,7 @@ func (c *sandboxEventStore) GetLastNEvents(sandboxId string, n int) ([]*SandboxE _, span := c.tracer.Start(c.ctx, "sandbox-event-get-last-n") defer span.End() - result, err := c.redisClient.ZRevRangeWithScores(c.ctx, sandboxId, 0, int64(n-1)).Result() + result, err := c.redisClient.ZRevRangeWithScores(c.ctx, EventPrefix+sandboxId, 0, int64(n-1)).Result() if err != nil { return nil, err } @@ -91,7 +115,7 @@ func (c *sandboxEventStore) AddEvent(sandboxId string, event *SandboxEvent, expi _, span := c.tracer.Start(c.ctx, "sandbox-event-store") defer span.End() - return c.redisClient.ZAdd(c.ctx, sandboxId, redis.Z{ + return c.redisClient.ZAdd(c.ctx, EventPrefix+sandboxId, redis.Z{ Score: float64(time.Now().UnixNano()), Member: event, }).Err() @@ -101,7 +125,7 @@ func (c *sandboxEventStore) DelEvent(sandboxId string) error { _, span := c.tracer.Start(c.ctx, "sandbox-event-delete") defer span.End() - return c.redisClient.Del(c.ctx, sandboxId).Err() + return c.redisClient.Del(c.ctx, EventPrefix+sandboxId).Err() } func (c *sandboxEventStore) Close() error { diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index f645313106..709fef772d 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -18,6 +18,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/event" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/fc" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/network" @@ -241,6 +242,7 @@ func ResumeSandbox( devicePool *nbd.DevicePool, allowInternet, useClickhouseMetrics bool, + eventStore event.SandboxEventStore, ) (*Sandbox, *Cleanup, error) { childCtx, childSpan := tracer.Start(ctx, "new-sandbox") defer childSpan.End() @@ -421,6 +423,12 @@ func ResumeSandbox( return nil, cleanup, fmt.Errorf("failed to wait for sandbox start: %w", err) } + sandboxIP := ips.slot.HostIPString() + eventStore.SetSandboxIP(config.SandboxId, sandboxIP) + cleanup.AddPriority(func(ctx context.Context) error { + return eventStore.DelSandboxIP(sandboxIP) + }) + go sbx.Checks.Start() return sbx, cleanup, nil diff --git a/packages/orchestrator/internal/server/main.go b/packages/orchestrator/internal/server/main.go index 4427a1a828..93bc3e08c1 100644 --- a/packages/orchestrator/internal/server/main.go +++ b/packages/orchestrator/internal/server/main.go @@ -12,6 +12,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/internal/grpcserver" "github.com/e2b-dev/infra/packages/orchestrator/internal/proxy" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/event" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/network" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/template" @@ -36,6 +37,7 @@ type server struct { devicePool *nbd.DevicePool persistence storage.StorageProvider featureFlags *featureflags.Client + eventStore event.SandboxEventStore } type Service struct { @@ -62,6 +64,7 @@ func New( proxy *proxy.SandboxProxy, sandboxes *smap.Map[*sandbox.Sandbox], featureFlags *featureflags.Client, + eventStore event.SandboxEventStore, ) (*Service, error) { srv := &Service{info: info} @@ -89,6 +92,7 @@ func New( devicePool: devicePool, persistence: persistence, featureFlags: featureFlags, + eventStore: eventStore, } meter := tel.MeterProvider.Meter("orchestrator.sandbox") diff --git a/packages/orchestrator/internal/server/sandboxes.go b/packages/orchestrator/internal/server/sandboxes.go index 467c5a4f1b..ed72532990 100644 --- a/packages/orchestrator/internal/server/sandboxes.go +++ b/packages/orchestrator/internal/server/sandboxes.go @@ -69,6 +69,7 @@ func (s *server) Create(ctxConn context.Context, req *orchestrator.SandboxCreate s.devicePool, config.AllowSandboxInternet, metricsWriteFlag, + s.eventStore, ) if err != nil { zap.L().Error("failed to create sandbox, cleaning up", zap.Error(err)) diff --git a/packages/orchestrator/main.go b/packages/orchestrator/main.go index c35bb04457..acb26515e3 100644 --- a/packages/orchestrator/main.go +++ b/packages/orchestrator/main.go @@ -246,6 +246,7 @@ func run(port, proxyPort, sbxEventServerPort uint) (success bool) { sbxEventHandlers := event.NewEventHandlers(ctx, eventStore) sbxEventServer := event.NewSandboxEventServer(sbxEventServerPort, sbxEventHandlers) + defer sbxEventServer.Close(ctx) networkPool, err := network.NewPool(ctx, tel.MeterProvider, network.NewSlotsPoolSize, network.ReusedSlotsPoolSize, clientID, tracer) if err != nil { @@ -271,7 +272,7 @@ func run(port, proxyPort, sbxEventServerPort uint) (success bool) { zap.L().Fatal("failed to create sandbox observer", zap.Error(err)) } - _, err = server.New(ctx, grpcSrv, tel, networkPool, devicePool, tracer, serviceInfo, sandboxProxy, sandboxes, featureFlags) + _, err = server.New(ctx, grpcSrv, tel, networkPool, devicePool, tracer, serviceInfo, sandboxProxy, sandboxes, featureFlags, eventStore) if err != nil { zap.L().Fatal("failed to create server", zap.Error(err)) } From 7535b6e2138a7c7cdb253c5918c9fa541bb2e133 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 8 Jul 2025 16:25:27 -0700 Subject: [PATCH 18/21] remove sandbox id middleware --- .../internal/sandbox/event/server.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/event/server.go b/packages/orchestrator/internal/sandbox/event/server.go index 44289265dd..a2efc61bcd 100644 --- a/packages/orchestrator/internal/sandbox/event/server.go +++ b/packages/orchestrator/internal/sandbox/event/server.go @@ -11,25 +11,11 @@ type SandboxEventServer struct { server *http.Server } -func validateHeaders(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - sandboxID := r.Header.Get("E2B_SANDBOX_ID") - teamID := r.Header.Get("E2B_TEAM_ID") - - if sandboxID == "" || teamID == "" { - http.Error(w, "missing required headers", http.StatusBadRequest) - return - } - - next.ServeHTTP(w, r) - } -} - func NewSandboxEventServer(port uint, handlers []EventHandler) *SandboxEventServer { mux := http.NewServeMux() for _, handler := range handlers { - mux.HandleFunc(handler.Path(), validateHeaders(handler.HandlerFunc)) + mux.HandleFunc(handler.Path(), handler.HandlerFunc) } server := &http.Server{ From d25a9f3698bb18419518d8b1fab3d05925354bd4 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 10 Jul 2025 16:34:05 -0700 Subject: [PATCH 19/21] use public ip address for sandbox event routing that is not meant to route according to RFC --- .../internal/sandbox/network/network_linux.go | 6 ++++-- packages/orchestrator/internal/template/build/rootfs.go | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 126d9d0758..360e8adeb1 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -10,6 +10,7 @@ import ( "runtime" "github.com/coreos/go-iptables/iptables" + "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "go.uber.org/zap" @@ -217,8 +218,9 @@ func (s *Slot) CreateNetwork() error { return fmt.Errorf("error creating postrouting rule: %w", err) } - // Redirect http://event.e2b.dev traffic destined to event server - err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", "8.8.8.7", "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") + // Redirect traffic destined to event server + eventIP := env.GetEnv("SANDBOX_EVENT_IP", "203.0.113.0") + err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") if err != nil { return fmt.Errorf("error creating HTTP redirect rule to sandbox event server: %w", err) } diff --git a/packages/orchestrator/internal/template/build/rootfs.go b/packages/orchestrator/internal/template/build/rootfs.go index d5927c42e7..ccb59de978 100644 --- a/packages/orchestrator/internal/template/build/rootfs.go +++ b/packages/orchestrator/internal/template/build/rootfs.go @@ -18,6 +18,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/oci" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/writer" artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry" + "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/e2b-dev/infra/packages/shared/pkg/storage" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -197,7 +198,9 @@ ExecStart=-/sbin/agetty --noissue --autologin root %I 115200,38400,9600 vt102 ` hostname := "e2b.local" - eventProxyHostname := "event.e2b.dev" + eventProxyHostname := "events.e2b.dev" + + eventIP := env.GetEnv("SANDBOX_EVENT_IP", "203.0.113.0") hosts := fmt.Sprintf(`127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -206,8 +209,8 @@ ff00:: ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters 127.0.1.1 %s -8.8.8.7 %s -`, hostname, eventProxyHostname) +%s %s +`, hostname, eventIP, eventProxyHostname) e2bFile := fmt.Sprintf(`ENV_ID=%s BUILD_ID=%s From 43091427da3c0260c49042c700a0a3df296d2f1e Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 10 Jul 2025 17:07:15 -0700 Subject: [PATCH 20/21] golint --- .../orchestrator/internal/sandbox/network/network_linux.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 360e8adeb1..23415812bf 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -10,10 +10,11 @@ import ( "runtime" "github.com/coreos/go-iptables/iptables" - "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "go.uber.org/zap" + + "github.com/e2b-dev/infra/packages/shared/pkg/env" ) func (s *Slot) CreateNetwork() error { From 78ffeba7eca16a0d06ae3c45cd37eb4ef14adb29 Mon Sep 17 00:00:00 2001 From: 0div Date: Fri, 11 Jul 2025 13:49:40 -0700 Subject: [PATCH 21/21] introduce internal const.go to orchestrator package --- packages/orchestrator/internal/consts.go | 13 +++++++++++++ .../internal/sandbox/network/network_linux.go | 4 ++-- .../orchestrator/internal/template/build/rootfs.go | 4 ++-- 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 packages/orchestrator/internal/consts.go diff --git a/packages/orchestrator/internal/consts.go b/packages/orchestrator/internal/consts.go new file mode 100644 index 0000000000..2d0e30657c --- /dev/null +++ b/packages/orchestrator/internal/consts.go @@ -0,0 +1,13 @@ +package internal + +import ( + "github.com/e2b-dev/infra/packages/shared/pkg/env" +) + +const ( + defaultSandboxEventIP = "203.0.113.0" +) + +func GetSandboxEventIP() string { + return env.GetEnv("SANDBOX_EVENT_IP", defaultSandboxEventIP) +} diff --git a/packages/orchestrator/internal/sandbox/network/network_linux.go b/packages/orchestrator/internal/sandbox/network/network_linux.go index 23415812bf..7987f2fb75 100644 --- a/packages/orchestrator/internal/sandbox/network/network_linux.go +++ b/packages/orchestrator/internal/sandbox/network/network_linux.go @@ -14,7 +14,7 @@ import ( "github.com/vishvananda/netns" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/shared/pkg/env" + "github.com/e2b-dev/infra/packages/orchestrator/internal" ) func (s *Slot) CreateNetwork() error { @@ -220,7 +220,7 @@ func (s *Slot) CreateNetwork() error { } // Redirect traffic destined to event server - eventIP := env.GetEnv("SANDBOX_EVENT_IP", "203.0.113.0") + eventIP := internal.GetSandboxEventIP() err = tables.Append("nat", "PREROUTING", "-i", s.VethName(), "-p", "tcp", "-d", eventIP, "--dport", "80", "-j", "REDIRECT", "--to-port", "5010") if err != nil { return fmt.Errorf("error creating HTTP redirect rule to sandbox event server: %w", err) diff --git a/packages/orchestrator/internal/template/build/rootfs.go b/packages/orchestrator/internal/template/build/rootfs.go index ccb59de978..19dbb846cc 100644 --- a/packages/orchestrator/internal/template/build/rootfs.go +++ b/packages/orchestrator/internal/template/build/rootfs.go @@ -14,11 +14,11 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "github.com/e2b-dev/infra/packages/orchestrator/internal" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/ext4" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/oci" "github.com/e2b-dev/infra/packages/orchestrator/internal/template/build/writer" artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry" - "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/e2b-dev/infra/packages/shared/pkg/storage" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -200,7 +200,7 @@ ExecStart=-/sbin/agetty --noissue --autologin root %I 115200,38400,9600 vt102 hostname := "e2b.local" eventProxyHostname := "events.e2b.dev" - eventIP := env.GetEnv("SANDBOX_EVENT_IP", "203.0.113.0") + eventIP := internal.GetSandboxEventIP() hosts := fmt.Sprintf(`127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback