Skip to content

Commit 449d4dd

Browse files
b41shb41sh
committed
Merge branch 'main' into feat-variant-ext-types
2 parents 83fa549 + 41f1ad1 commit 449d4dd

File tree

13 files changed

+151
-51
lines changed

13 files changed

+151
-51
lines changed

‎.github/actions/pack_deb/action.yml

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ runs:
4747
aarch64)
4848
echo "deb_arch=arm64" >> $GITHUB_OUTPUT
4949
;;
50+
*)
51+
echo "Unsupported arch: ${{ inputs.arch }}"
52+
exit 1
53+
;;
5054
esac
5155
deb_version=${version/-/.}
5256
echo "deb_version=${deb_version/v/}" >> $GITHUB_OUTPUT
@@ -59,32 +63,26 @@ runs:
5963
- name: Build Databend Query Package
6064
shell: bash
6165
run: |
62-
export path="distro"
63-
export version="${{ inputs.version }}"
64-
export deb_version="${{ steps.info.outputs.deb_version }}"
65-
export deb_arch="${{ steps.info.outputs.deb_arch }}"
66-
pkg_name="databend-query_${deb_version}_${deb_arch}.deb"
67-
nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${deb_arch} ${deb_version} ${path}' < scripts/distribution/nfpm-query.yaml)
66+
yq -i '.arch = "${{ steps.info.outputs.deb_arch }}"' scripts/distribution/nfpm-query.yaml
67+
yq -i '.version = "${{ steps.info.outputs.deb_version }}"' scripts/distribution/nfpm-query.yaml
68+
pkg_name="databend-query_${{ steps.info.outputs.deb_version }}_${{ steps.info.outputs.deb_arch }}.deb"
69+
nfpm pkg --packager deb -t "distro/${pkg_name}" -f scripts/distribution/nfpm-query.yaml
6870
6971
- name: Build Databend Meta Package
7072
shell: bash
7173
run: |
72-
export path="distro"
73-
export version="${{ inputs.version }}"
74-
export deb_version="${{ steps.info.outputs.deb_version }}"
75-
export deb_arch="${{ steps.info.outputs.deb_arch }}"
76-
pkg_name="databend-meta_${deb_version}_${deb_arch}.deb"
77-
nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${deb_arch} ${deb_version} ${path}' < scripts/distribution/nfpm-meta.yaml)
74+
yq -i '.arch = "${{ steps.info.outputs.deb_arch }}"' scripts/distribution/nfpm-meta.yaml
75+
yq -i '.version = "${{ steps.info.outputs.deb_version }}"' scripts/distribution/nfpm-meta.yaml
76+
pkg_name="databend-meta_${{ steps.info.outputs.deb_version }}_${{ steps.info.outputs.deb_arch }}.deb"
77+
nfpm pkg --packager deb -t "distro/${pkg_name}" -f scripts/distribution/nfpm-meta.yaml
7878
7979
- name: Build Databend Debug Package
8080
shell: bash
8181
run: |
82-
export path="distro"
83-
export version="${{ inputs.version }}"
84-
export deb_version="${{ steps.info.outputs.deb_version }}"
85-
export deb_arch="${{ steps.info.outputs.deb_arch }}"
86-
pkg_name="databend-query-dbg_${deb_version}_${deb_arch}.deb"
87-
nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${deb_arch} ${deb_version} ${path}' < scripts/distribution/nfpm-query-dbg.yaml)
82+
yq -i '.arch = "${{ steps.info.outputs.deb_arch }}"' scripts/distribution/nfpm-query-dbg.yaml
83+
yq -i '.version = "${{ steps.info.outputs.deb_version }}"' scripts/distribution/nfpm-query-dbg.yaml
84+
pkg_name="databend-query-dbg_${{ steps.info.outputs.deb_version }}_${{ steps.info.outputs.deb_arch }}.deb"
85+
nfpm pkg --packager deb -t "distro/${pkg_name}" -f scripts/distribution/nfpm-query-dbg.yaml
8886
8987
- name: Update release to github
9088
shell: bash

‎.github/scripts/generate_release_note.js

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
const fs = require("fs");
2+
3+
function escapeHtml(text) {
4+
const map = {
5+
"&": "&amp;",
6+
"<": "&lt;",
7+
">": "&gt;",
8+
'"': "&quot;",
9+
"'": "&#039;",
10+
};
11+
return text.replace(/[&<>"']/g, (m) => map[m]);
12+
}
13+
14+
module.exports = async ({ github, context, core }) => {
15+
const { VERSION, DATE } = process.env;
16+
fs.mkdirSync("docs/release-stable", { recursive: true });
17+
const df = `docs/release-stable/${DATE}_${VERSION}.md`;
18+
19+
const releases = await github.rest.repos.listReleases({
20+
owner: "databendlabs",
21+
repo: "databend",
22+
});
23+
const release = releases.data.find((r) => r.name === VERSION);
24+
if (!release) {
25+
core.setFailed(`Release ${VERSION} not found`);
26+
return;
27+
}
28+
29+
let body = release.body;
30+
body = body.split("\n").slice(1).join("\n");
31+
body = "---\n" + body;
32+
body = body.replace(/^--$/gm, "---");
33+
body = body.replace(/^asset:.*$/gm, "");
34+
body = body.replace(
35+
/https:\/\/github\.com\/databendlabs\/databend\/pull\/([0-9]+)/g,
36+
"[#$1](https://github.com/databendlabs/databend/pull/$1)"
37+
);
38+
body = escapeHtml(body);
39+
40+
fs.writeFileSync(df, body);
41+
};

‎.github/workflows/release.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,16 @@ jobs:
9292
shell: bash
9393
run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
9494
- name: Generate Release Note
95+
uses: actions/github-script@v7
9596
env:
96-
GH_TOKEN: ${{ github.token }}
97+
VERSION: ${{ needs.create_release.outputs.version }}
98+
DATE: ${{ steps.date.outputs.DATE }}
99+
with:
100+
script: |
101+
const script = require('./.github/scripts/generate_release_note.js')
102+
await script({ github, context, core })
103+
- name: Add Release Note
97104
run: |
98-
mkdir -p docs/release-stable
99-
df="docs/release-stable/${{ steps.date.outputs.DATE }}_${{ needs.create_release.outputs.version }}.md"
100-
echo "---" > $df
101-
gh release view --repo databendlabs/databend ${{ needs.create_release.outputs.version }} >> $df
102-
sed -i -E 's/^--$/---/g' $df
103-
sed -i -E '/^asset:/d' $df
104-
sed -i -E 's_https://github.com/databendlabs/databend/pull/([0-9]+)_[#\1](https://github.com/databendlabs/databend/pull/\1)_g' $df
105105
git add docs/release-stable
106106
git status
107107
- uses: peter-evans/create-pull-request@v4

‎scripts/distribution/nfpm-meta.yaml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: "databend-meta"
2-
arch: "${deb_arch}"
2+
arch: "${arch}"
33
platform: "linux"
4-
version: "${deb_version}"
4+
version: "${version}"
55
section: "database"
66
priority: "extra"
77
maintainer: "Databend Labs <opensource@databend.com>"
@@ -15,20 +15,20 @@ depends:
1515
- libc6 (>= 2.31)
1616
contents:
1717
# Binaries
18-
- src: ${path}/bin/databend-meta
18+
- src: distro/bin/databend-meta
1919
dst: /usr/bin/databend-meta
20-
- src: ${path}/bin/databend-metactl
20+
- src: distro/bin/databend-metactl
2121
dst: /usr/bin/databend-metactl
2222

2323
# Configs
24-
- src: ${path}/configs/databend-meta.toml
24+
- src: distro/configs/databend-meta.toml
2525
dst: /etc/databend/databend-meta.toml
2626
type: config
2727

2828
# Systemd
29-
- src: ${path}/systemd/databend-meta.service
29+
- src: distro/systemd/databend-meta.service
3030
dst: /lib/systemd/system/databend-meta.service
31-
- src: ${path}/systemd/databend-meta.default
31+
- src: distro/systemd/databend-meta.default
3232
dst: /etc/default/databend-meta
3333

3434
- dst: /var/lib/databend
@@ -41,5 +41,5 @@ contents:
4141
mode: 0755
4242

4343
scripts:
44-
preinstall: ${path}/scripts/preinstall.sh
45-
postinstall: ${path}/scripts/postinstall.sh
44+
preinstall: distro/scripts/preinstall.sh
45+
postinstall: distro/scripts/postinstall.sh

‎scripts/distribution/nfpm-query-dbg.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: "databend-query-dbg"
2-
arch: "${deb_arch}"
2+
arch: "${arch}"
33
platform: "linux"
4-
version: "${deb_version}"
4+
version: "${version}"
55
section: "database"
66
priority: "extra"
77
maintainer: "Databend Labs <opensource@databend.com>"
@@ -14,5 +14,5 @@ depends:
1414
- databend-query
1515
contents:
1616
# Binaries
17-
- src: ${path}/bin/databend-query.debug
17+
- src: distro/bin/databend-query.debug
1818
dst: /usr/bin/databend-query.debug

‎scripts/distribution/nfpm-query.yaml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: "databend-query"
2-
arch: "${deb_arch}"
2+
arch: "${arch}"
33
platform: "linux"
4-
version: "${deb_version}"
4+
version: "${version}"
55
section: "database"
66
priority: "extra"
77
maintainer: "Databend Labs <opensource@databend.com>"
@@ -15,22 +15,22 @@ depends:
1515
- libc6 (>= 2.31)
1616
contents:
1717
# Binaries
18-
- src: ${path}/bin/databend-query
18+
- src: distro/bin/databend-query
1919
dst: /usr/bin/databend-query
2020

2121
# Configs
22-
- src: ${path}/configs/databend-query.toml
22+
- src: distro/configs/databend-query.toml
2323
dst: /etc/databend/databend-query.toml
2424
type: config
2525

2626
# Systemd
27-
- src: ${path}/systemd/databend-query.service
27+
- src: distro/systemd/databend-query.service
2828
dst: /lib/systemd/system/databend-query.service
29-
- src: ${path}/systemd/databend-query.default
29+
- src: distro/systemd/databend-query.default
3030
dst: /etc/default/databend-query
3131

3232
# Docs
33-
- src: ${path}/readme.txt
33+
- src: distro/readme.txt
3434
dst: /usr/share/doc/databend/readme.txt
3535
file_info:
3636
mode: 0644
@@ -45,5 +45,5 @@ contents:
4545
mode: 0755
4646

4747
scripts:
48-
preinstall: ${path}/scripts/preinstall.sh
49-
postinstall: ${path}/scripts/postinstall.sh
48+
preinstall: distro/scripts/preinstall.sh
49+
postinstall: distro/scripts/postinstall.sh

‎src/meta/semaphore/src/acquirer/acquirer.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,11 @@ impl Acquirer {
150150
// Step 4: Wait for the semaphore to be acquired or removed.
151151

152152
while let Some(sem_event) = self.permit_event_rx.recv().await {
153-
debug!("semaphore event: {:?}", sem_event);
153+
info!(
154+
"Acquirer({}): received semaphore event: {:?}",
155+
self.ctx, sem_event
156+
);
157+
154158
match sem_event {
155159
PermitEvent::Acquired((seq, _)) => {
156160
if seq == permit_key.seq {

‎src/meta/semaphore/src/meta_event_subscriber/processor.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use codeq::Decode;
1616
use databend_common_meta_types::protobuf::WatchResponse;
1717
use databend_common_meta_types::SeqV;
18+
use log::info;
1819
use log::warn;
1920
use tokio::sync::mpsc;
2021

@@ -74,6 +75,11 @@ impl Processor {
7475
prev: Option<SeqV<PermitEntry>>,
7576
current: Option<SeqV<PermitEntry>>,
7677
) -> Result<(), ConnectionClosed> {
78+
info!(
79+
"{} processing kv change: {}: {:?} -> {:?}",
80+
self.ctx, sem_key, prev, current
81+
);
82+
7783
// Update local queue to update the acquired/released state.
7884
let state_changes = match (prev, current) {
7985
(None, Some(entry)) => self.queue.insert(sem_key.seq, entry.data),
@@ -89,7 +95,11 @@ impl Processor {
8995
}
9096
};
9197

98+
info!("{} queue state: {}", self.ctx, self.queue);
99+
92100
for event in state_changes {
101+
info!("{} sending event: {}", self.ctx, event);
102+
93103
self.tx.send(event).await.map_err(|e| {
94104
ConnectionClosed::new_str(format!("Semaphore-Watcher fail to send {}", e.0))
95105
.context(&self.ctx)

‎src/meta/semaphore/src/meta_event_subscriber/subscriber.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@ impl MetaEventSubscriber {
7474
.context(&self.ctx)
7575
})?;
7676

77+
info!(
78+
"{} watch stream created: [{}, {})",
79+
self.ctx, self.left, self.right
80+
);
81+
7782
Ok(strm)
7883
}
7984

@@ -116,6 +121,11 @@ impl MetaEventSubscriber {
116121
}
117122
};
118123

124+
info!(
125+
"{} received event from watch-stream: {:?}",
126+
self.ctx, watch_result
127+
);
128+
119129
let Some(watch_response) = watch_result? else {
120130
// TODO: add retry connecting.
121131
error!("watch-stream closed: {}", self.ctx);

‎src/meta/semaphore/src/queue/semaphore_queue.rs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414

1515
use std::collections::BTreeMap;
16+
use std::fmt;
1617

1718
use crate::queue::semaphore_event::PermitEvent;
1819
use crate::PermitEntry;
@@ -32,6 +33,27 @@ pub struct SemaphoreQueue {
3233
waiting: BTreeMap<PermitSeq, PermitEntry>,
3334
}
3435

36+
impl fmt::Display for SemaphoreQueue {
37+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
38+
write!(f, "SemaphoreQueue{{ {}/{}", self.size, self.capacity,)?;
39+
40+
write!(f, ", acquired: [")?;
41+
for (seq, entry) in &self.acquired {
42+
write!(f, "{}:{} ", seq, entry)?;
43+
}
44+
write!(f, "]")?;
45+
46+
write!(f, ", waiting: [")?;
47+
for (seq, entry) in &self.waiting {
48+
write!(f, "{}:{} ", seq, entry)?;
49+
}
50+
write!(f, "]")?;
51+
52+
write!(f, "}}")?;
53+
Ok(())
54+
}
55+
}
56+
3557
impl SemaphoreQueue {
3658
pub fn new(capacity: u64) -> Self {
3759
SemaphoreQueue {
@@ -152,6 +174,21 @@ mod tests {
152174
use crate::queue::*;
153175
use crate::PermitEntry;
154176

177+
#[test]
178+
fn test_display() {
179+
let queue = SemaphoreQueue {
180+
size: 10,
181+
capacity: 20,
182+
acquired: BTreeMap::from([(1, ent("t1", 3)), (2, ent("t2", 4))]),
183+
waiting: BTreeMap::from([(3, ent("t3", 5)), (4, ent("t4", 6))]),
184+
};
185+
186+
assert_eq!(
187+
format!("{}", queue),
188+
"SemaphoreQueue{ 10/20, acquired: [1:PermitEntry(id:t1, n:3) 2:PermitEntry(id:t2, n:4) ], waiting: [3:PermitEntry(id:t3, n:5) 4:PermitEntry(id:t4, n:6) ]}"
189+
);
190+
}
191+
155192
#[test]
156193
fn test_insert() {
157194
// Test case 1: Insert when there is enough capacity

0 commit comments

Comments
 (0)