Add GitHub Copilot Connection and associated dashboards (#8728)

* feat(gh-copilot): add backend plugin

* feat(config-ui): add gh-copilot connection UI

* feat(grafana): add GitHub Copilot dashboards

* fix: gh-copilot CI fixes

Author: ewega

.gitignore

@@ -150,6 +150,9 @@ bin
 libgit2
 .air.toml
 
+# Playwright CLI snapshots/logs (local dev)
+.playwright-cli/
+
 # auto generated code
 backend/mocks/
 backend/server/api/docs/swagger.json

backend/plugins/gh-copilot/README.md

@@ -0,0 +1,124 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# GitHub Copilot Plugin (Adoption Metrics)
+
+This plugin ingests GitHub Copilot **organization-level adoption metrics** (daily usage and seat assignments) and provides a Grafana dashboard for adoption trends.
+
+It follows the same structure/patterns as other DevLake data-source plugins (notably `backend/plugins/q_dev`).
+
+## What it collects
+
+**Phase 1 endpoints** (GitHub Copilot REST API):
+
+- `GET /orgs/{org}/copilot/billing`
+- `GET /orgs/{org}/copilot/billing/seats`
+- `GET /orgs/{org}/copilot/metrics`
+
+**Stored data (tool layer)**:
+
+- `_tool_copilot_org_metrics` (daily aggregates)
+- `_tool_copilot_language_metrics` (editor/language breakdown)
+- `_tool_copilot_seats` (seat assignments)
+
+## Data flow (high level)
+
+```mermaid
+flowchart LR
+	GH[GitHub Copilot REST API]
+	RAW[(Raw tables\n_raw_copilot_*)]
+	TOOL[(Tool tables\n_tool_copilot_*)]
+	GRAF[Grafana Dashboard\nGitHub Copilot Adoption]
+
+	GH --> RAW --> TOOL --> GRAF
+```
+
+## Repository layout
+
+- `api/` – REST layer for connections/scopes
+- `impl/` – plugin meta, options, connection helpers
+- `models/` – tool-layer models + migrations
+- `tasks/` – collectors/extractors and pipeline registration
+- `e2e/` – E2E fixtures and golden CSV assertions
+- `docs/` – documentation assets
+
+## Setup
+
+### Prerequisites
+
+- GitHub Copilot Business or Enterprise enabled for the target organization
+- A token that can access GitHub Copilot billing/metrics (classic PAT with `manage_billing:copilot` works)
+
+### 1) Create a connection
+
+1. DevLake UI → **Data Integrations → Add Connection → GitHub Copilot**
+2. Fill in:
+	 - **Name**: e.g. `GitHub Copilot Octodemo`
+	 - **Endpoint**: defaults to `https://api.github.com`
+	 - **Organization**: GitHub org slug
+	 - **Token**: PAT with required scope
+3. Click **Test Connection** (calls `GET /orgs/{org}/copilot/billing`).
+4. Save the connection.
+
+### 2) Create a scope
+
+Add an organization scope for that connection. For Phase 1, `implementationDate` is optional.
+
+### 3) Create a blueprint (recipe)
+
+Use a blueprint plan like:
+
+```json
+[
+	[
+		{
+			"plugin": "gh-copilot",
+			"options": {
+				"connectionId": 1,
+				"scopeId": "octodemo"
+			}
+		}
+	]
+]
+```
+
+Run the blueprint daily to keep metrics up to date.
+
+## Dashboard
+
+The Grafana dashboard JSON is in `grafana/dashboards/copilot/adoption.json`.
+
+Link: `grafana/dashboards/copilot/adoption.json`
+
+## Error handling guidance
+
+- **403 Forbidden** → token missing required billing/metrics scope, or org lacks GitHub Copilot access
+- **404 Not Found** → incorrect org slug, or GitHub Copilot endpoints unavailable for the org
+- **422 Unprocessable Entity** → GitHub Copilot metrics disabled in GitHub org settings
+
+- **429 Too Many Requests** → respect `Retry-After`; collectors implement backoff/retry
+
+Tokens are sanitized before persisting. When patching an existing connection, omit the token to retain the encrypted value already stored in DevLake.
+
+## Limitations (Phase 1)
+
+- Metrics endpoint is limited to a rolling **100-day** window (GitHub API constraint)
+- GitHub enforces a privacy threshold (often **≥ 5 engaged users**) and may omit daily data
+- Enterprise download endpoints and per-user metrics (JSONL exports) are intentionally deferred to Phase 2+
+
+## More docs
+
+- Spec quickstart: `specs/001-copilot-metrics-plugin/quickstart.md`

backend/plugins/gh-copilot/api/blueprint_v200.go

@@ -0,0 +1,87 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/plugin"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+	"github.com/apache/incubator-devlake/plugins/gh-copilot/models"
+	"github.com/apache/incubator-devlake/plugins/gh-copilot/tasks"
+)
+
+// MakeDataSourcePipelinePlanV200 generates the pipeline plan for blueprint v2.0.0.
+func MakeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	// load connection and scopes from the db
+	_, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	// map blueprint scopes (scopeId/scopeConfigId) to concrete stored scopes
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Copilot metrics are org-level and currently don't map to a standard domain-layer top-level entity.
+	// Return an empty scope list to avoid adding meaningless project mappings.
+	return plan, nil, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.GhCopilotScope, models.GhCopilotScopeConfig],
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+
+		scope := scopeDetail.Scope
+		task, err := helper.MakePipelinePlanTask(
+			"gh-copilot",
+			subtaskMetas,
+			nil,
+			tasks.GhCopilotOptions{
+				ConnectionId: scope.ConnectionId,
+				ScopeId:      scope.Id,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		stage = append(stage, task)
+		plan[i] = stage
+	}
+
+	return plan, nil
+}

backend/plugins/gh-copilot/api/connection.go

@@ -0,0 +1,106 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/gh-copilot/models"
+)
+
+// PostConnections creates a new Copilot connection.
+func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.GhCopilotConnection{}
+	if err := helper.Decode(input.Body, connection, vld); err != nil {
+		return nil, err
+	}
+
+	connection.Normalize()
+	if err := validateConnection(connection); err != nil {
+		return nil, err
+	}
+
+	if err := connectionHelper.Create(connection, input); err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil
+}
+
+func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.GhCopilotConnection{}
+	if err := connectionHelper.First(connection, input.Params); err != nil {
+		return nil, err
+	}
+	if err := (&models.GhCopilotConnection{}).MergeFromRequest(connection, input.Body); err != nil {
+		return nil, errors.Convert(err)
+	}
+	connection.Normalize()
+	if err := validateConnection(connection); err != nil {
+		return nil, err
+	}
+	if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil
+}
+
+func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	conn := &models.GhCopilotConnection{}
+	output, err := connectionHelper.Delete(conn, input)
+	if err != nil {
+		return output, err
+	}
+	output.Body = conn.Sanitize()
+	return output, nil
+}
+
+func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	var connections []models.GhCopilotConnection
+	if err := connectionHelper.List(&connections); err != nil {
+		return nil, err
+	}
+	for i := range connections {
+		connections[i] = connections[i].Sanitize()
+	}
+	return &plugin.ApiResourceOutput{Body: connections}, nil
+}
+
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection := &models.GhCopilotConnection{}
+	if err := connectionHelper.First(connection, input.Params); err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil
+}
+
+func validateConnection(connection *models.GhCopilotConnection) errors.Error {
+	if connection == nil {
+		return errors.BadInput.New("connection is required")
+	}
+	if connection.Organization == "" && !connection.HasEnterprise() {
+		return errors.BadInput.New("either enterprise or organization is required")
+	}
+	if connection.Token == "" {
+		return errors.BadInput.New("token is required")
+	}
+	if connection.RateLimitPerHour < 0 {
+		return errors.BadInput.New("rateLimitPerHour must be non-negative")
+	}
+	return nil
+}