Skip to content

Commit 682e4a1

Browse files
feat: mirgator plugin (#5347)
* Add files via upload * Update 257_migration_plugin.up.sql * Add files via upload * Delete assets/plugin-migrate-logo.png * Add files via upload * Delete assets/plugin-migrate-logo.png * Add files via upload * Delete assets/plugin-migrate-logo.png * Add files via upload * Delete assets/plugin-migrate.png * added logo for go plugin * Add files via upload * Update 257_migration_plugin.up.sql * Update 257_migration_plugin.down.sql * Update 257_migration_plugin.up.sql * Rename 257_migration_plugin.up.sql to 259_migration_plugin.up.sql * Rename 257_migration_plugin.down.sql to 259_migration_plugin.down.sql * Update 259_migration_plugin.down.sql * Update 259_migration_plugin.up.sql * Update 259_migration_plugin.up.sql * Update golang-migrate.md * Add files via upload * Delete assets/plugin-golang-migrator.png * Add files via upload
1 parent 050e4b4 commit 682e4a1

File tree

4 files changed

+173
-0
lines changed

4 files changed

+173
-0
lines changed

assets/plugin-golang-migrate.png

19.3 KB
Loading
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# GoLang-migrate
2+
3+
Migrate reads migrations from sources file and applies them in correct order to a database.
4+
5+
**Prerequisite**: Make sure you have SQL files in format used by the golang-migrate tool.
6+
7+
**official-documentation**: https://github.com/golang-migrate/migrate
8+
**postgres-example**: https://github.com/golang-migrate/migrate/tree/master/database/postgres
9+
10+
1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). or
11+
2. Click **+ Add task**.
12+
3. Select **GoLang-migrate** from **PRESET PLUGINS**.
13+
14+
15+
* Enter a relevant name in the `Task name` field. It is a mandatory field.
16+
* Enter a descriptive message for the task in the `Description` field. It is an optional field.
17+
* Provide a value for the input variable.<br> Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.</br>
18+
19+
| Variable | Format | Description |
20+
| ---- | ---- | ---- |
21+
| DB_TYPE | String | Currently this plugin support postgres,mongodb,mongodb+srv,mysql,sqlserver. |
22+
| DB_HOST | String | The hostname, service endpoint or IP address of the database server. |
23+
| DB_PORT | String | The port number on which the database server is listening. |
24+
| DB_NAME | String | The name of the specific database instance you want to connect to. |
25+
| DB_USER | String | The username required to authenticate to the database.|
26+
| DB_PASSWORD | String | The password required to authenticate to the database. |
27+
| SCRIPT_LOCATION | String | Location of SQL files that need to be run on desired database. |
28+
| MIGRATE_IMAGE | String | Docker image of golang-migrate default:migrate/migrate. |
29+
| MIGRATE_TO_VERSION | String | migrate to which version of sql script need to be run on desired database(default: 0 is for all files in directory). |
30+
| PARAM | String | extra params that runs with db queries. |
31+
| POST_COMMAND | String | post commands that runs at the end of script. |
32+
33+
* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:<ul><li>`Set trigger conditions` or</li><li>`Set skip conditions`</li></ul>
34+
35+
* Click **Update Pipeline**.
36+
37+
38+
### Notes
39+
- Use `in-cluster/ Execute tasks in application environment` feature in `pre-deploy` or `post-deploy`, in case when the database service is not reachable or accessible from devtron cluster.
40+
- In case the `DB_TYPE` is not supported for your database, then use `POST_COMMAND` as
41+
42+
```
43+
POST_COMMAND:
44+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database <myDB-connection-string>" goto $MIGRATE_TO_VERSION;
45+
```
46+
- use `DB_PASSWORD` with `scope-variable` feature for more security.
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
DELETE FROM pipeline_stage_step_variable where pipeline_stage_step_id in (select id from pipeline_stage_step where name ='GoLang-migrate');
2+
DELETE FROM plugin_step_variable where plugin_step_id=(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false);
3+
DELETE FROM plugin_stage_mapping where plugin_id=(SELECT id from plugin_metadata where name='GoLang-migrate');
4+
DELETE FROM plugin_step where plugin_id=(SELECT id FROM plugin_metadata WHERE name='GoLang-migrate');
5+
DELETE FROM plugin_tag_relation WHERE plugin_id=(SELECT id FROM plugin_metadata WHERE name='GoLang-migrate');
6+
DELETE FROM pipeline_stage_step WHERE name ='GoLang-migrate';
7+
DELETE FROM plugin_metadata where name='GoLang-migrate';
8+
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
INSERT INTO plugin_metadata (id,name,description,type,icon,deleted,created_on,created_by,updated_on,updated_by)
2+
VALUES (nextval('id_seq_plugin_metadata'),'GoLang-migrate','This plugin provides a seamless way to manage and execute database migrations, ensuring your database schema evolves safely and predictably with your application code. ','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/plugin-golang-migrate.png',false,'now()',1,'now()',1);
3+
4+
INSERT INTO plugin_stage_mapping (id,plugin_id,stage_type,created_on,created_by,updated_on,updated_by)
5+
VALUES (nextval('id_seq_plugin_stage_mapping'),(SELECT id from plugin_metadata where name='GoLang-migrate'), 0,'now()',1,'now()',1);
6+
7+
INSERT INTO "plugin_pipeline_script" ("id", "script","type","deleted","created_on", "created_by", "updated_on", "updated_by")
8+
VALUES (
9+
nextval('id_seq_plugin_pipeline_script'),
10+
$$#!/bin/sh
11+
set -e
12+
set -o pipefail
13+
if [ -z $DB_HOST ]; then
14+
echo "Please enter DB_HOST"
15+
exit 1
16+
fi
17+
if [ -z $DB_NAME ]; then
18+
echo "Please enter DB_NAME"
19+
exit 1
20+
fi
21+
22+
# Verify that you are on the correct branch and commit
23+
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
24+
CURRENT_COMMIT=$(git rev-parse HEAD)
25+
if [ -z $SCRIPT_LOCATION ]; then
26+
echo "we are running migration on current location"
27+
pwd
28+
SCRIPT_LOCATION="."
29+
fi
30+
echo "Current branch: $CURRENT_BRANCH"
31+
echo "Current commit: $CURRENT_COMMIT"
32+
echo "Migrating to version (0 for latest)"
33+
echo $MIGRATE_TO_VERSION;
34+
35+
DB_CRED=""
36+
if [ -n "$DB_USER" ]; then
37+
DB_CRED="$DB_USER:$DB_PASSWORD@"
38+
fi
39+
40+
if [ "$DB_TYPE" = "postgres" ]; then
41+
echo "migration for postgres"
42+
if [ $MIGRATE_TO_VERSION -eq "0" ]; then
43+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database postgres://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" up;
44+
else
45+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database postgres://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" goto $MIGRATE_TO_VERSION;
46+
fi
47+
docker run -v "$PWD:$PWD" "$MIGRATE_IMAGE" -path "$PWD/$SCRIPT_LOCATION" -database "postgres://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?$PARAM" version > migration-golang-current-version.txt 2>&1
48+
elif [ "$DB_TYPE" = "mongodb" ]; then
49+
echo "migration for mongodb"
50+
if [ $MIGRATE_TO_VERSION -eq "0" ]; then
51+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mongodb://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" up;
52+
else
53+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mongodb://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" goto $MIGRATE_TO_VERSION;
54+
fi
55+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mongodb://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" version > migration-golang-current-version.txt 2>&1
56+
elif [ "$DB_TYPE" = "mongodb+srv" ]; then
57+
echo "migration for mongodb"
58+
if [ $MIGRATE_TO_VERSION -eq "0" ]; then
59+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mongodb+srv://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" up;
60+
else
61+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mongodb+srv://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" goto $MIGRATE_TO_VERSION;
62+
fi
63+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mongodb+srv://$DB_CRED$DB_HOST:$DB_PORT/$DB_NAME?"$PARAM" version > migration-golang-current-version.txt 2>&1
64+
elif [ "$DB_TYPE" = "mysql" ]; then
65+
echo "migration for mysql"
66+
DB="tcp($DB_HOST:$DB_PORT)"
67+
if [ $MIGRATE_TO_VERSION -eq "0" ]; then
68+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mysql://$DB_CRED$DB/$DB_NAME?"$PARAM" up;
69+
else
70+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mysql://$DB_CRED$DB/$DB_NAME?"$PARAM" goto $MIGRATE_TO_VERSION;
71+
fi
72+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database mysql://$DB_CRED$DB/$DB_NAME?"$PARAM" version > migration-golang-current-version.txt 2>&1
73+
elif [ "$DB_TYPE" = "sqlserver" ]; then
74+
echo "migration for sqlserver"
75+
if [ $MIGRATE_TO_VERSION -eq "0" ]; then
76+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database sqlserver://$DB_CRED$DB_HOST:$DB_PORT?"$PARAM" up;
77+
else
78+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database sqlserver://$DB_CRED$DB_HOST:$DB_PORT?"$PARAM" goto $MIGRATE_TO_VERSION;
79+
fi
80+
docker run -v $PWD:$PWD $MIGRATE_IMAGE -path $PWD/$SCRIPT_LOCATION -database sqlserver://$DB_CRED$DB_HOST:$DB_PORT?"$PARAM" version > migration-golang-current-version.txt 2>&1
81+
else
82+
echo "no database matched"
83+
fi
84+
$POST_COMMAND
85+
export POST_MIGRATION_VERION=$(cat migration-golang-current-version.txt)
86+
if [ -z $POST_MIGRATION_VERION ]; then
87+
POST_MIGRATION_VERION="0"
88+
fi
89+
echo "migration completed"$$,
90+
'SHELL',
91+
'f',
92+
'now()',
93+
1,
94+
'now()',
95+
1
96+
);
97+
98+
99+
100+
101+
102+
103+
INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by")
104+
VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='GoLang-migrate'),'Step 1','Step 1 - GoLang-migrate','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1);
105+
106+
107+
INSERT INTO plugin_step_variable (id,plugin_step_id,name,format, description,is_exposed,allow_empty_value,default_value,value,variable_type,value_type,previous_step_index,variable_step_index,variable_step_index_in_plugin,reference_variable_name,deleted,created_on,created_by,updated_on,updated_by)VALUES
108+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'DB_TYPE','STRING','Currently this plugin support postgres,mongodb,mongodb+srv,mysql,sqlserver.','t','f',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
109+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'DB_HOST','STRING','The hostname endpoint or IP address of the database server.','t','f',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
110+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'DB_PORT','STRING','The port number on which the database server is listening.','t','f',null,null,'INPUT','NEW',null,1,null,null, 'f','now()',1,'now()',1),
111+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'DB_NAME','STRING','The name of the specific database instance you want to connect to.','t','f',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
112+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'DB_USER','STRING','The username required to authenticate to the database.','t','f',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
113+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'DB_PASSWORD','STRING','The password required to authenticate to the database.','t','f',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
114+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'SCRIPT_LOCATION','STRING','sql files location in git repo','t','t',null,null,'INPUT','NEW',null,1,null,null, 'f','now()',1,'now()',1),
115+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'MIGRATE_IMAGE','STRING','Docker image of golang-migrate default:migrate/migrate','t','t','migrate/migrate',null,'INPUT','NEW',null,1,null,null, 'f','now()',1,'now()',1),
116+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'MIGRATE_TO_VERSION','STRING','migrate to which version of sql script (default: 0 is for all files in directory)','t','f','0',null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
117+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'PARAM','STRING','Additional connection parameters (optional), typically specified as key-value pairs. example: `sslmode=disable`', 't','t',-1,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
118+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'POST_COMMAND','STRING','post commands that runs at the end of script','t','t',null,null,'INPUT','NEW',null,1,null,null, 'f','now()',1,'now()',1),
119+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='GoLang-migrate' and ps."index"=1 and ps.deleted=false),'POST_MIGRATION_VERION','STRING','migration version after running the SQL files', 't','t',-1,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1);

0 commit comments

Comments
 (0)