Update README and other doc. (#269)

Author: paulosjca

.gitignore

@@ -5,6 +5,8 @@
 *.so
 *.dylib
 bin/*
+dashboard/containers/**
+dashboard/bin/*
 **/main
 **/*.out
 

README.md

@@ -15,31 +15,29 @@ The main executable is a [custom controller] that manages resources of kind
 [LoadTest]. This controller must be deployed to the cluster before load tests
 can be run on it. The controller is implemented with [kubebuilder].
 
-There is also a set of tools used to prepare prebuilt images and run batches of
-tests. These tools are used to generate the dashboard linked from the [gRPC
-performance benchmarking] page. For more information, see
-[tools](tools/README.md).
+There is also a set of [tools](tools/README.md) used to generate load test
+configurations, prepare prebuilt images and run batches of tests. These tools
+are used to run batches of tests for continuous integration.
 
 [Examples](config/samples/README.md) of load test configurations in the
 supported languages are also provided.
 
 [custom controller]: cmd/controller/main.go
-[grpc performance benchmarking]: https://grpc.io/docs/guides/benchmarking/
 [kubebuilder]: https://kubebuilder.io
 [loadtest]: config/crd/bases/e2etest.grpc.io_loadtests.yaml
 
-## Contributing
+## Dashboard
 
-Welcome! Please read [how to contribute](CONTRIBUTING.md) before proceeding.
+The data generated in continuous integration are saved to [BigQuery], and
+displayed on a public dashboard linked from the [gRPC performance benchmarking]
+page.
 
-This project includes third party dependencies as git submodules. Be sure to
-initialize and update them when setting up a development environment:
+For more information, and to build your own dashboard, see
+[dashboard](dashboard/README.md).
 
-```shell
-# Init/update during the clone
-git clone --recursive https://github.com/grpc/test-infra.git  # HTTPS
-git clone --recursive git@github.com:grpc/test-infra.git      # SSH
+[bigquery]: https://cloud.google.com/bigquery
+[grpc performance benchmarking]: https://grpc.io/docs/guides/benchmarking/
 
-# (or) Init/update after the clone
-git submodule update --init
-```
+## Contributing
+
+Welcome! Please read [how to contribute](CONTRIBUTING.md) before proceeding.

dashboard/Makefile

@@ -1,25 +1,28 @@
 GOCMD ?= go
-GOARGS= -trimpath
-REPLICATOR_OUTPUT_DIR ?= bin
+GOARGS = -trimpath
 CONFIG_TEMPLATE_DIR ?= config
+DASHBOARDS_CONFIG_DIR ?= $(CONFIG_TEMPLATE_DIR)/grafana/dashboards/default
+REPLICATOR_CONFIG_TEMPLATE ?= $(CONFIG_TEMPLATE_DIR)/postgres_replicator/default/config.yaml
+REPLICATOR_OUTPUT_DIR ?= bin
 GRAFANA_CONTAINER_OUTPUT_DIR ?= containers/grafana
 REPLICATOR_CONTAINER_OUTPUT_DIR ?= containers/replicator
 
 configure-grafana:
 	scripts/check_env.sh GCP_GRAFANA_SERVICE PG_USER PG_PASS PG_DATABASE GRAFANA_ADMIN_PASS CLOUD_SQL_INSTANCE
 	mkdir -p $(GRAFANA_CONTAINER_OUTPUT_DIR)
-	cp -r $(CONFIG_TEMPLATE_DIR)/grafana/dashboards/default $(GRAFANA_CONTAINER_OUTPUT_DIR)/dashboards
+	cp -r $(DASHBOARDS_CONFIG_DIR) $(GRAFANA_CONTAINER_OUTPUT_DIR)/dashboards
 	cp -r grafana/* $(GRAFANA_CONTAINER_OUTPUT_DIR)
 	cp $(CONFIG_TEMPLATE_DIR)/grafana/Dockerfile $(GRAFANA_CONTAINER_OUTPUT_DIR)/Dockerfile
 	envsubst < $(CONFIG_TEMPLATE_DIR)/grafana/app.yaml > $(GRAFANA_CONTAINER_OUTPUT_DIR)/app.yaml
 
 configure-replicator:
 	scripts/check_env.sh GCP_PROJECT_ID GCP_DATA_TRANSFER_SERVICE BQ_PROJECT_ID PG_USER PG_PASS PG_DATABASE CLOUD_SQL_INSTANCE
-	mkdir -p containers/replicator
+	mkdir -p $(REPLICATOR_CONTAINER_OUTPUT_DIR)
+	envsubst < $(REPLICATOR_CONFIG_TEMPLATE) > $(REPLICATOR_CONTAINER_OUTPUT_DIR)/config.yaml
 	cp $(CONFIG_TEMPLATE_DIR)/postgres_replicator/Dockerfile $(REPLICATOR_CONTAINER_OUTPUT_DIR)/Dockerfile
 	envsubst < $(CONFIG_TEMPLATE_DIR)/postgres_replicator/app.yaml > $(REPLICATOR_CONTAINER_OUTPUT_DIR)/app.yaml
-	envsubst < $(CONFIG_TEMPLATE_DIR)/postgres_replicator/default/config.yaml > $(REPLICATOR_CONTAINER_OUTPUT_DIR)/config.yaml
 
+# Replicator target is for testing only. Production binary is built by Docker.
 replicator: fmt vet
 	$(GOCMD) build $(GOARGS) -o $(REPLICATOR_OUTPUT_DIR)/replicator cmd/postgres_replicator/main.go
 

dashboard/README.md

@@ -0,0 +1,76 @@
+# Dashboard
+
+This folder contains the components necessary to build and deploy a dashboard to
+visualize gRPC OSS benchmarking results.
+
+gRPC OSS benchmarks save results to [BigQuery]. The dashboard consists of two
+components:
+
+1. A [Postgres replicator], to transfer the results to a Postgres database.
+1. A Grafana dashboard, to displays the results from the Postgres database.
+
+These components can be built and deployed manually using the
+[Makefile](Makefile) (see [manual build](#manual-build)).
+
+Notice that the dashboard build is independent from the top-level build.
+
+## Configuration
+
+The configuration of the Postgres replicator is defined in a YAML file. The
+default configuration is defined here, in template form:
+
+- [config/postgres_replicator/default/config.yaml][postgres replicator config]
+
+For more information, see [Postgres replicator].
+
+The configuration of the Grafana dashboard is defined in a set of dashboards
+specified in JSON files. The default configuration is defined here:
+
+- [config/grafana/dashboards/default/][grafana dashboard config]
+
+The continuous integration dashboard linked from the [gRPC performance
+benchmarking] page uses the default configuration. The variables
+`REPLICATOR_CONFIG_TEMPLATE` and `DASHBOARDS_CONFIG_DIR` can be set to build
+dashboards with different configurations.
+
+[bigquery]: https://cloud.google.com/bigquery
+[grafana dashboard config]: config/grafana/dashboards/default/
+[grpc performance benchmarking]: https://grpc.io/docs/guides/benchmarking/
+[postgres replicator]: cmd/postgres_replicator/README.md
+[postgres replicator config]: config/postgres_replicator/default/config.yaml
+
+## Manual build
+
+Several environment variables must be set before building and deploying. The
+table below shows the names and values of the variables in our main dashboard:
+
+| Variable                    | Value                                         |
+| --------------------------- | --------------------------------------------- |
+| `BQ_PROJECT_ID`             | `grpc-testing`                                |
+| `CLOUD_SQL_INSTANCE`        | `grpc-testing:us-central1:grafana-datasource` |
+| `GCP_DATA_TRANSFER_SERVICE` | `postgres-replicator`                         |
+| `GCP_GRAFANA_SERVICE`       | `grafana`                                     |
+| `GCP_PROJECT_ID`            | `grpc-testing`                                |
+| `GRAFANA_ADMIN_PASS`        | \*\*\*                                        |
+| `PG_DATABASE`               | `datasource`                                  |
+| `PG_PASS`                   | \*\*\*                                        |
+| `PG_USER`                   | `grafana-user`                                |
+
+Docker files that can be used to build and deploy the Postgres replicator and
+Grafana dashboard are then created with the following commands:
+
+```shell
+make configure-replicator
+make configure-dashboard
+```
+
+## Cloud build
+
+The continuous integration dashboard is built and deployed with [Cloud Build],
+using the configuration specified in [cloudbuild.yaml](cloudbuild.yaml).
+
+The use of Cloud Build allows the dashboard to be redeployed automatically on
+configuration changes. In addition, it allows passwords such as `PG_PASS` and
+`GRAFANA_ADMIN_PASS` to be stored as secrets in the cloud project.
+
+[cloud build]: https://cloud.google.com/build

dashboard/cmd/postgres_replicator/README.md

@@ -4,7 +4,7 @@ This tool replicates streaming data from BigQuery into PostgreSQL.
 
 ## Configuration
 
-It is configured with a YAML file. Here is an example configuration file:
+The tool is configured with a YAML file. Here is an example configuration file:
 
 ```yaml
 # Source database settings
@@ -52,7 +52,7 @@ This port number can be overridden via the `PORT` environment variable.
 
 ## Running
 
-From the test-infra project root, run `make replicator`, then
+From the dashboard project root, run `make replicator`, then
 `bin/replicator -c <config_file>`.
 
 When the replicator receives a `GET` request for `/run`, it will transfer new
@@ -64,8 +64,8 @@ ignore additional requests to `/run` (but still return `200`).
 1. The data in the database must be sequentially ordered by time. Specifically,
    there must be a column of BigQuery datatype TIMESTAMP. This timestamp should
    strictly increase for new data.
-2. Configuration only allows for one GCP project at a time.
-3. All table names must be unique, even across multiple BigQuery datasets
+1. Configuration only allows for one GCP project at a time.
+1. All table names must be unique, even across multiple BigQuery datasets
 
 ## How it works
 
@@ -87,13 +87,14 @@ BigQuery table.
 
 For non-nested columns, the replicator currently supports
 
-- FLOAT
-- STRING
-- TIMESTAMP
+- `FLOAT`
+- `STRING`
+- `TIMESTAMP`
 
 BigQuery table fields that are of the `RECORD`/`REPEATED`/`STRUCT`/`ARRAY` type
 are converted to JSON and stored in Postgres as the JSON datatype. To retrieve
-values from the Postgres table, see the [JSON Functions and Operators] page.
+values from the Postgres table, see the documentation on [JSON Functions and
+Operators].
 
 For example, if your BigQuery table has the FLOAT field `stats.client1.latency`,
 this could then be queried and typecast with the following in Postgres:

dashboard/config/postgres_replicator/Dockerfile

@@ -21,8 +21,8 @@ ARG REPOSITORY=grpc/test-infra
 ARG GITREF=master
 
 RUN git clone https://github.com/$REPOSITORY.git src \
- && cd src/dashboard && git checkout $GITREF \
- && make replicator REPLICATOR_OUTPUT_DIR=/
+    && cd src/dashboard && git checkout $GITREF \
+    && make replicator REPLICATOR_OUTPUT_DIR=/
 
 # Copy replicator binary and run it
 FROM golang:1.16

dashboard/scripts/check_env.sh

@@ -2,15 +2,16 @@
 
 # check_env checks that all required environment variables are set
 check_env () {
-  env_vars=("$@")
-  missing_var=0
+  local -a env_vars=( "$@" )
+  local missing_var=0
+  local var
   for var in "${env_vars[@]}"; do
     if [ -z "${!var}" ]; then
       echo "${var} not set"
       missing_var=1
     fi
   done
-  return $missing_var
+  return ${missing_var}
 }
 
 check_env "$@"