From 37d33fdd710ee1e168c16d6320ddf08c39d7f12e Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 9 Jul 2022 19:00:07 -0400 Subject: [PATCH] Internals + blog PR This automatically opens an internals thread and a blog PR after promoting a stable release (if configured, and will only be configured on dev-static). Caveat: needs human user to set release notes URL and scheduled release date. That will be done via flags to start-release.py in simpleinfra, probably. --- src/config.rs | 90 ++++++++++++++++++++++ src/curl_helper.rs | 68 +++++++++++++++++ src/discourse.rs | 69 +++++++++++++++++ src/github.rs | 184 ++++++++++++++++++++++++++------------------- src/main.rs | 82 ++++++++++++++++++++ 5 files changed, 415 insertions(+), 78 deletions(-) create mode 100644 src/curl_helper.rs create mode 100644 src/discourse.rs diff --git a/src/config.rs b/src/config.rs index af6443d..9ffe562 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,3 +1,4 @@ +use crate::discourse::Discourse; use crate::github::Github; use crate::Context; use anyhow::{Context as _, Error}; @@ -113,6 +114,27 @@ pub(crate) struct Config { /// Should be a org/repo code, e.g., rust-lang/rust. pub(crate) rustc_tag_repository: Option, + /// Where to publish new blog PRs. + /// + /// We create a new PR announcing releases in this repository; currently we + /// don't automatically merge it (but that might change in the future). + /// + /// Should be a org/repo code, e.g., rust-lang/blog.rust-lang.org. + pub(crate) blog_repository: Option, + + /// The expected release date, for the blog post announcing dev-static + /// releases. Expected to be in YYYY-MM-DD format. + /// + /// This is used to produce the expected release date in blog posts and to + /// generate the release notes URL (targeting stable branch on + /// rust-lang/rust). + pub(crate) scheduled_release_date: Option, + + /// These are Discourse configurations for where to post dev-static + /// announcements. Currently we only post dev release announcements. + pub(crate) discourse_api_key: Option, + pub(crate) discourse_api_user: Option, + /// This is a github app private key, used for the release steps which /// require action on GitHub (e.g., kicking off a new thanks GHA build, /// opening pull requests against the blog for dev releases, promoting @@ -151,6 +173,10 @@ impl Config { upload_dir: require_env("UPLOAD_DIR")?, wip_recompress: bool_env("WIP_RECOMPRESS")?, rustc_tag_repository: maybe_env("RUSTC_TAG_REPOSITORY")?, + blog_repository: maybe_env("BLOG_REPOSITORY")?, + scheduled_release_date: maybe_env("BLOG_SCHEDULED_RELEASE_DATE")?, + discourse_api_user: maybe_env("DISCOURSE_API_USER")?, + discourse_api_key: maybe_env("DISCOURSE_API_KEY")?, github_app_key: maybe_env("GITHUB_APP_KEY")?, github_app_id: maybe_env("GITHUB_APP_ID")?, }) @@ -163,6 +189,70 @@ impl Config { None } } + pub(crate) fn discourse(&self) -> Option { + if let (Some(key), Some(user)) = (&self.discourse_api_key, &self.discourse_api_user) { + Some(Discourse::new( + "https://internals.rust-lang.org".to_owned(), + user.clone(), + key.clone(), + )) + } else { + None + } + } + + pub(crate) fn blog_contents( + &self, + release: &str, + archive_date: &str, + for_blog: bool, + internals_url: Option<&str>, + ) -> Option { + let scheduled_release_date = self.scheduled_release_date?; + let release_notes_url = format!( + "https://github.com/rust-lang/rust/blob/stable/RELEASES.md#version-{}-{}", + release.replace('.', ""), + scheduled_release_date.format("%Y-%m-%d"), + ); + let human_date = scheduled_release_date.format("%B %d"); + let internals = internals_url + .map(|url| format!("You can leave feedback on the [internals thread]({url}).")) + .unwrap_or_default(); + let prefix = if for_blog { + format!( + r#"--- +layout: post +title: "{} pre-release testing" +author: Release automation +team: The Release Team +---{}"#, + release, "\n\n", + ) + } else { + String::new() + }; + Some(format!( + "{prefix}The {release} pre-release is ready for testing. The release is scheduled for +{human_date}. [Release notes can be found here.][relnotes] + +You can try it out locally by running: + +```plain +RUSTUP_DIST_SERVER=https://dev-static.rust-lang.org rustup update stable +``` + +The index is . + +{internals} + +The release team is also thinking about changes to our pre-release process: +we'd love your feedback [on this GitHub issue][feedback]. + +[relnotes]: {release_notes_url} +[feedback]: https://github.com/rust-lang/release-team/issues/16 + " + )) + } } fn maybe_env(name: &str) -> Result, Error> diff --git a/src/curl_helper.rs b/src/curl_helper.rs new file mode 100644 index 0000000..30052ec --- /dev/null +++ b/src/curl_helper.rs @@ -0,0 +1,68 @@ +use anyhow::Context; +use curl::easy::Easy; + +pub trait BodyExt { + fn with_body(&mut self, body: S) -> Request<'_, S>; + fn without_body(&mut self) -> Request<'_, ()>; +} + +impl BodyExt for Easy { + fn with_body(&mut self, body: S) -> Request<'_, S> { + Request { + body: Some(body), + client: self, + } + } + fn without_body(&mut self) -> Request<'_, ()> { + Request { + body: None, + client: self, + } + } +} + +pub struct Request<'a, S> { + body: Option, + client: &'a mut Easy, +} + +impl Request<'_, S> { + pub fn send_with_response(self) -> anyhow::Result { + use std::io::Read; + let mut response = Vec::new(); + let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); + { + let mut transfer = self.client.transfer(); + // The unwrap in the read_function is basically guaranteed to not + // happen: reading into a slice can't fail. We can't use `?` since the + // return type inside transfer isn't compatible with io::Error. + if let Some(mut body) = body.as_deref() { + transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; + } + transfer.write_function(|new_data| { + response.extend_from_slice(new_data); + Ok(new_data.len()) + })?; + transfer.perform()?; + } + serde_json::from_slice(&response) + .with_context(|| format!("{}", String::from_utf8_lossy(&response))) + } + + pub fn send(self) -> anyhow::Result<()> { + use std::io::Read; + let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); + { + let mut transfer = self.client.transfer(); + // The unwrap in the read_function is basically guaranteed to not + // happen: reading into a slice can't fail. We can't use `?` since the + // return type inside transfer isn't compatible with io::Error. + if let Some(mut body) = body.as_deref() { + transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; + } + transfer.perform()?; + } + + Ok(()) + } +} diff --git a/src/discourse.rs b/src/discourse.rs new file mode 100644 index 0000000..bd2b9c1 --- /dev/null +++ b/src/discourse.rs @@ -0,0 +1,69 @@ +use crate::curl_helper::BodyExt; +use curl::easy::Easy; + +pub struct Discourse { + root: String, + api_key: String, + api_username: String, + client: Easy, +} + +impl Discourse { + pub fn new(root: String, api_username: String, api_key: String) -> Discourse { + Discourse { + root, + api_key, + api_username, + client: Easy::new(), + } + } + + fn start_new_request(&mut self) -> anyhow::Result<()> { + self.client.reset(); + self.client.useragent("rust-lang/promote-release")?; + let mut headers = curl::easy::List::new(); + headers.append(&format!("Api-Key: {}", self.api_key))?; + headers.append(&format!("Api-Username: {}", self.api_username))?; + headers.append("Content-Type: application/json")?; + self.client.http_headers(headers)?; + Ok(()) + } + + /// Returns a URL to the topic + pub fn create_topic( + &mut self, + category: u32, + title: &str, + body: &str, + ) -> anyhow::Result { + #[derive(serde::Serialize)] + struct Request<'a> { + title: &'a str, + #[serde(rename = "raw")] + body: &'a str, + category: u32, + archetype: &'a str, + } + #[derive(serde::Deserialize)] + struct Response { + topic_id: u32, + topic_slug: String, + } + self.start_new_request()?; + self.client.post(true)?; + self.client.url(&format!("{}/posts.json", self.root))?; + let resp = self + .client + .with_body(Request { + title, + body, + category, + archetype: "regular", + }) + .send_with_response::()?; + Ok(format!( + "{}/t/{}/{}", + self.root, resp.topic_slug, resp.topic_id + )) + } +} diff --git a/src/github.rs b/src/github.rs index de7ce80..cfdc3fc 100644 --- a/src/github.rs +++ b/src/github.rs @@ -1,4 +1,4 @@ -use anyhow::Context; +use crate::curl_helper::BodyExt; use curl::easy::Easy; use rsa::pkcs1::DecodeRsaPrivateKey; use sha2::Digest; @@ -138,13 +138,6 @@ impl RepositoryClient<'_> { email: &'a str, } - #[derive(serde::Serialize)] - struct CreateRefInternal<'a> { - #[serde(rename = "ref")] - ref_: &'a str, - sha: &'a str, - } - #[derive(serde::Deserialize)] struct CreatedTag { sha: String, @@ -170,6 +163,40 @@ impl RepositoryClient<'_> { }) .send_with_response::()?; + self.create_ref(&format!("refs/tags/{}", tag.tag_name), &created.sha)?; + + Ok(()) + } + + /// Returns the SHA of the tip of this ref, if it exists. + pub(crate) fn get_ref(&mut self, name: &str) -> anyhow::Result { + // This mostly exists to make sure the request is successful rather than + // really checking the created ref (which we already know). + #[derive(serde::Deserialize)] + struct Reference { + object: Object, + } + #[derive(serde::Deserialize)] + struct Object { + sha: String, + } + + self.start_new_request()?; + self.github.client.get(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/git/ref/{name}", + repository = self.repo, + ))?; + Ok(self + .github + .client + .without_body() + .send_with_response::()? + .object + .sha) + } + + pub(crate) fn create_ref(&mut self, name: &str, sha: &str) -> anyhow::Result<()> { // This mostly exists to make sure the request is successful rather than // really checking the created ref (which we already know). #[derive(serde::Deserialize)] @@ -178,6 +205,13 @@ impl RepositoryClient<'_> { #[allow(unused)] ref_: String, } + #[derive(serde::Serialize)] + struct CreateRefInternal<'a> { + #[serde(rename = "ref")] + name: &'a str, + sha: &'a str, + } + self.start_new_request()?; self.github.client.post(true)?; self.github.client.url(&format!( @@ -186,10 +220,7 @@ impl RepositoryClient<'_> { ))?; self.github .client - .with_body(CreateRefInternal { - ref_: &format!("refs/tags/{}", tag.tag_name), - sha: &created.sha, - }) + .with_body(CreateRefInternal { name, sha }) .send_with_response::()?; Ok(()) @@ -215,6 +246,69 @@ impl RepositoryClient<'_> { Ok(()) } + + /// Note that this API *will* fail if the file already exists in this + /// branch; we don't update existing files. + pub(crate) fn create_file( + &mut self, + branch: &str, + path: &str, + content: &str, + ) -> anyhow::Result<()> { + #[derive(serde::Serialize)] + struct Request<'a> { + message: &'a str, + content: &'a str, + branch: &'a str, + } + self.start_new_request()?; + self.github.client.put(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/contents/{path}", + repository = self.repo, + ))?; + self.github + .client + .with_body(Request { + branch, + message: "Creating file via promote-release automation", + content: &base64::encode(&content), + }) + .send()?; + Ok(()) + } + + pub(crate) fn create_pr( + &mut self, + base: &str, + head: &str, + title: &str, + body: &str, + ) -> anyhow::Result<()> { + #[derive(serde::Serialize)] + struct Request<'a> { + head: &'a str, + base: &'a str, + title: &'a str, + body: &'a str, + } + self.start_new_request()?; + self.github.client.post(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/pulls", + repository = self.repo, + ))?; + self.github + .client + .with_body(Request { + base, + head, + title, + body, + }) + .send()?; + Ok(()) + } } #[derive(Copy, Clone)] @@ -225,69 +319,3 @@ pub(crate) struct CreateTag<'a> { pub(crate) tagger_name: &'a str, pub(crate) tagger_email: &'a str, } - -trait BodyExt { - fn with_body(&mut self, body: S) -> Request<'_, S>; - fn without_body(&mut self) -> Request<'_, ()>; -} - -impl BodyExt for Easy { - fn with_body(&mut self, body: S) -> Request<'_, S> { - Request { - body: Some(body), - client: self, - } - } - fn without_body(&mut self) -> Request<'_, ()> { - Request { - body: None, - client: self, - } - } -} - -struct Request<'a, S> { - body: Option, - client: &'a mut Easy, -} - -impl Request<'_, S> { - fn send_with_response(self) -> anyhow::Result { - use std::io::Read; - let mut response = Vec::new(); - let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); - { - let mut transfer = self.client.transfer(); - // The unwrap in the read_function is basically guaranteed to not - // happen: reading into a slice can't fail. We can't use `?` since the - // return type inside transfer isn't compatible with io::Error. - if let Some(mut body) = body.as_deref() { - transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; - } - transfer.write_function(|new_data| { - response.extend_from_slice(new_data); - Ok(new_data.len()) - })?; - transfer.perform()?; - } - serde_json::from_slice(&response) - .with_context(|| format!("{}", String::from_utf8_lossy(&response))) - } - - fn send(self) -> anyhow::Result<()> { - use std::io::Read; - let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); - { - let mut transfer = self.client.transfer(); - // The unwrap in the read_function is basically guaranteed to not - // happen: reading into a slice can't fail. We can't use `?` since the - // return type inside transfer isn't compatible with io::Error. - if let Some(mut body) = body.as_deref() { - transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; - } - transfer.perform()?; - } - - Ok(()) - } -} diff --git a/src/main.rs b/src/main.rs index 53cd3ca..2579576 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,8 @@ mod build_manifest; mod config; +mod curl_helper; +mod discourse; mod github; mod sign; mod smoke_test; @@ -28,6 +30,8 @@ use crate::config::{Channel, Config}; const TARGET: &str = env!("TARGET"); +const BLOG_PRIMARY_BRANCH: &str = "master"; + struct Context { work: PathBuf, handle: Easy, @@ -223,6 +227,11 @@ impl Context { // around. let _ = fs::remove_dir_all(&self.dl_dir()); + // This opens a PR and starts an internals thread announcing a + // stable dev-release (we distinguish dev by the presence of metadata + // which lets us know where to create and what to put in the blog). + self.open_blog()?; + // We do this last, since it triggers triagebot posting the GitHub // release announcement (and since this is not actually really // important). @@ -672,6 +681,79 @@ impl Context { Ok(()) } + fn open_blog(&mut self) -> Result<(), Error> { + // We rely on the blog variables not being set in production to disable + // blogging on the actual release date. + if self.config.channel != Channel::Stable { + eprintln!("Skipping blogging -- not on stable"); + return Ok(()); + } + + let mut github = if let Some(github) = self.config.github() { + github + } else { + eprintln!("Skipping blogging - GitHub credentials not configured"); + return Ok(()); + }; + let mut discourse = if let Some(discourse) = self.config.discourse() { + discourse + } else { + eprintln!("Skipping blogging - Discourse credentials not configured"); + return Ok(()); + }; + let repository_for_blog = if let Some(repo) = &self.config.blog_repository { + repo.as_str() + } else { + eprintln!("Skipping blogging - blog repository not configured"); + return Ok(()); + }; + + let version = self.current_version.as_ref().expect("has current version"); + let internals_contents = + if let Some(contents) = self.config.blog_contents(version, &self.date, false, None) { + contents + } else { + eprintln!("Skipping internals - insufficient information to create blog post"); + return Ok(()); + }; + + let announcements_category = 18; + let internals_url = discourse.create_topic( + announcements_category, + &format!("Rust {} pre-release testing", version), + &internals_contents, + )?; + let blog_contents = if let Some(contents) = + self.config + .blog_contents(version, &self.date, true, Some(&internals_url)) + { + contents + } else { + eprintln!("Skipping blogging - insufficient information to create blog post"); + return Ok(()); + }; + + // Create a new branch so that we don't need to worry about the file + // already existing. In practice this *could* collide, but after merging + // a PR branches should get deleted, so it's very unlikely. + let name = format!("automation-{:x}", rand::random::()); + let mut token = github.token(repository_for_blog)?; + let master_sha = token.get_ref(&format!("heads/{BLOG_PRIMARY_BRANCH}"))?; + token.create_ref(&format!("refs/heads/{name}"), &master_sha)?; + token.create_file( + &name, + &format!( + "posts/inside-rust/{}-{}-prerelease.md", + chrono::Utc::today().format("%Y-%m-%d"), + version, + ), + &blog_contents, + )?; + token.create_pr(BLOG_PRIMARY_BRANCH, &name, "Pre-release announcement", "")?; + + Ok(()) + } + fn dl_dir(&self) -> PathBuf { self.work.join("dl") }