Upgrades (#51)

increment ubuntu container version
move from pygithub to ghapi
skip the chown step

Author: jacobmsft

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04 AS codeql_base
+FROM ubuntu:22.04 AS codeql_base
 LABEL maintainer="Github codeql team"
 
 # tzdata install needs to be non-interactive
@@ -12,33 +12,31 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \
     apt-get update && \
     apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
-    software-properties-common \
-    nodejs \
-    vim \
-    curl \
-    wget \
-    git \
-    build-essential \
-    unzip \
-    apt-transport-https \
-    python3.8 \
-    python3-venv \
-    python3-pip \
-    python3-setuptools \
-    python3-dev \
-    gnupg \
-    g++ \
-    make \
-    gcc \
-    apt-utils \
-    rsync \
-    file \
-    dos2unix \
-    gettext && \
-    apt-get clean && \
-    rm -f /usr/bin/python /usr/bin/pip && \
-    ln -s /usr/bin/python3.8 /usr/bin/python && \
-    ln -s /usr/bin/pip3 /usr/bin/pip 
+    	software-properties-common \
+        nodejs \
+    	vim \
+    	curl \
+    	wget \
+    	git \
+    	build-essential \
+    	unzip \
+    	apt-transport-https \
+        python3.10 \
+    	python3-venv \
+    	python3-pip \
+    	python3-setuptools \
+        python3-dev \
+        python-is-python3 \
+    	gnupg \
+    	g++ \
+    	make \
+    	gcc \
+    	apt-utils \
+        rsync \
+    	file \
+        dos2unix \
+    	gettext && \
+        apt-get clean
 
 # Install .NET Core and Java for tools/builds
 RUN cd /tmp && \
@@ -48,15 +46,13 @@ RUN cd /tmp && \
     apt-get install -y default-jdk apt-transport-https && \
     apt-get update && \
     rm packages-microsoft-prod.deb
-RUN apt-get install -y dotnet-sdk-3.1
+RUN apt-get install -y dotnet-sdk-6.0
 
 # Clone our setup and run scripts
-#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts
 RUN mkdir -p /usr/local/startup_scripts
-RUN ls -al /usr/local/startup_scripts
 COPY container /usr/local/startup_scripts/
-RUN pip3 install --upgrade pip \
-    && pip3 install -r /usr/local/startup_scripts/requirements.txt
+
+RUN pip3 install -r /usr/local/startup_scripts/requirements.txt
 
 # Install latest codeQL
 
@@ -83,8 +79,8 @@ RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql-
 ENV PYTHONIOENCODING=utf-8
 
 # Change ownership of all files and directories within CODEQL_HOME to the codeql user
-RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME}
+#RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME}
 
 USER ${USERNAME}
 
-ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"]
+ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] 

container/get-latest-codeql-version.py

@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 # get the parent directory of the script, to link libs
-
-import os
-import sys
-
 from libs.github import get_latest_github_repo_version
 
 def main():
-    latest_release = get_latest_github_repo_version("github/codeql-cli-binaries")
-    print(latest_release.title)
+    latest_release = get_latest_github_repo_version("github", "codeql-cli-binaries")
+    print(latest_release.tag_name)
 
 main()

container/libs/codeql.py

@@ -34,15 +34,15 @@ def download_and_install_latest_codeql(self, github_version):
         download_url = None
         download_path = None
         if os_name == 'posix':
-            download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-linux64.zip'
+            download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-linux64.zip'
             download_path = f'{self.TEMP_DIR}/codeql_linux.zip'
         elif os_name == 'nt':
-            download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-win64.zip'
+            download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-win64.zip'
             download_path = f'{self.TEMP_DIR}/codeql_windows.zip'
         else:
             exit(self.ERROR_UNKNOWN_OS)
 
-        logger.info(f'Downloading codeql-cli version {github_version.title}...')
+        logger.info(f'Downloading codeql-cli version {github_version.tag_name}...')
         check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8")
         self.install_codeql_cli(download_path)
         #rm /tmp/codeql_linux.zip
@@ -77,7 +77,7 @@ def get_current_local_version(self):
         return version
 
     def get_latest_codeql_github_version(self):
-        return get_latest_github_repo_version("github/codeql-cli-binaries")
+        return get_latest_github_repo_version("github", "codeql-cli-binaries")
 
     def install_codeql_cli(self, download_path):
         logger.info("Installing codeql-cli...")

container/libs/github.py

@@ -1,24 +1,26 @@
 import os
 from datetime import datetime, MINYEAR
-from github import Github, GitRelease, Repository, GithubException
+from ghapi.all import GhApi
+from datetime import datetime, timezone
+from dateutil import parser
 
-def get_latest_github_repo_version(repo):
+def get_latest_github_repo_version(owner, repository):
     # check for a github token that may be used alongside the codeql cli to upload github results
     # this will limit rate limting 403 errors on checking codeql versions, as the request will be authenticated if possible.
     # by default codeql uses env var "GITHUB_TOKEN" to authenticate
     # https://codeql.github.com/docs/codeql-cli/manual/github-upload-results/
     access_token = os.getenv('GITHUB_TOKEN')
-    client = Github(access_token) if access_token != None else Github()
-    repo = client.get_repo(repo)
-    releases = repo.get_releases()
+    api = GhApi(owner=owner, repo=repository, token=access_token) if access_token != None else GhApi(owner=owner, repo=repository)
+    releases = api.repos.list_releases()
     latest_release = get_latest_github_release(releases)
     return latest_release
 
 def get_latest_github_release(releases):
     latest_release = None
-    latest_date = datetime(MINYEAR, 1, 1)
+    latest_date = datetime(MINYEAR, 1, 1).replace(tzinfo=timezone.utc)
     for release in releases:
-        if release.created_at > latest_date:
-            latest_date = release.created_at
+        release_date = parser.parse(release.created_at)
+        if release_date > latest_date:
+            latest_date = release_date
             latest_release = release
     return latest_release

container/requirements.txt

@@ -1 +1,2 @@
-PyGithub==1.43.7
+ghapi==1.0.3
+python-dateutil==2.8.2

container/setup.py

@@ -46,7 +46,7 @@ def get_latest_codeql(args):
     # ensure we only query for the latest codeql cli version if we might actually update it
     if args.check_latest_cli:
         latest_online_version = codeql.get_latest_codeql_github_version()
-        if current_installed_version != latest_online_version.title:
+        if current_installed_version != latest_online_version.tag_name:
             # we got a newer version online, download and install it
             codeql.download_and_install_latest_codeql(latest_online_version)
     # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?)

container/startup.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import os
-import sys
 from time import sleep
 from libs.utils import get_env_variable, check_output_wrapper, get_logger
 from libs.codeql import *