chore(ci): Add kokoro pipeline for release and docs (#14)

* chore(ci): Add kokoro pipeline for release and docs * Update publish-docs.sh
googleapis · Feb 19, 2024 · a546889 · a546889
1 parent 37aab3e
commit a546889
Show file tree

Hide file tree

Showing 14 changed files with 1,479 additions and 0 deletions.
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
@@ -0,0  1,59 @@
 #!/bin/bash
 # Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 set -eo pipefail
 
 if [[ -z "${PROJECT_ROOT:-}" ]]; then
     PROJECT_ROOT="github/langchain-google-cloud-sql-mssql-python"
 fi
 
 cd "${PROJECT_ROOT}"
 
 # Disable buffering, so that the logs stream through.
 export PYTHONUNBUFFERED=1
 
 # Debug: show build environment
 env | grep KOKORO
 
 # Setup service account credentials.
 export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
 
 # Setup project id.
 export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
 
 # Remove old nox
 python3 -m pip uninstall --yes --quiet nox-automation
 
 # Install nox
 python3 -m pip install --upgrade --quiet nox
 python3 -m nox --version
 
 # If this is a continuous build, send the test log to the FlakyBot.
 # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
 if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
   cleanup() {
     chmod  x $KOKORO_GFILE_DIR/linux_amd64/flakybot
     $KOKORO_GFILE_DIR/linux_amd64/flakybot
   }
   trap cleanup EXIT HUP
 fi
 
 # If NOX_SESSION is set, it only runs the specified session,
 # otherwise run all the sessions.
 if [[ -n "${NOX_SESSION:-}" ]]; then
     python3 -m nox -s ${NOX_SESSION:-}
 else
     python3 -m nox
 fi
diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile
@@ -0,0  1,88 @@
 # Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from ubuntu:22.04
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # Ensure local Python is preferred over distribution Python.
 ENV PATH /usr/local/bin:$PATH
 
 # Install dependencies.
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
     apt-transport-https \
     build-essential \
     ca-certificates \
     curl \
     dirmngr \
     git \
     gpg-agent \
     graphviz \
     libbz2-dev \
     libdb5.3-dev \
     libexpat1-dev \
     libffi-dev \
     liblzma-dev \
     libreadline-dev \
     libsnappy-dev \
     libssl-dev \
     libsqlite3-dev \
     portaudio19-dev \
     python3-distutils \
     redis-server \
     software-properties-common \
     ssh \
     sudo \
     tcl \
     tcl-dev \
     tk \
     tk-dev \
     uuid-dev \
     wget \
     zlib1g-dev \
   && add-apt-repository universe \
   && apt-get update \
   && apt-get -y install jq \
   && apt-get clean autoclean \
   && apt-get autoremove -y \
   && rm -rf /var/lib/apt/lists/* \
   && rm -f /var/cache/apt/archives/*.deb
 
 ENV VERSION v4.2.0
 ENV BINARY yq_linux_amd64
 RUN wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq && \
     chmod  x /usr/bin/yq
 
 ###################### Install python 3.9.13
 
 # Download python 3.9.13
 RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz
 
 # Extract files
 RUN tar -xvf Python-3.9.13.tgz
 
 # Install python 3.9.13
 RUN ./Python-3.9.13/configure --enable-optimizations
 RUN make altinstall
 
 ###################### Install pip
 RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
   && python3 /tmp/get-pip.py \
   && rm /tmp/get-pip.py
 
 # Test pip
 RUN python3 -m pip
 
 CMD ["python3.8"]
diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg
@@ -0,0  1,67 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
 # Build logs will be here
 action {
   define_artifacts {
     regex: "**/*sponge_log.xml"
   }
 }
 
 # Download trampoline resources.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
 build_file: "langchain-google-cloud-sql-mssql-python/.kokoro/trampoline_v2.sh"
 
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
     key: "TRAMPOLINE_IMAGE"
     value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
 }
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/langchain-google-cloud-sql-mssql-python/.kokoro/publish-docs.sh"
 }
 
 env_vars: {
     key: "STAGING_BUCKET"
     value: "docs-staging"
 }
 
 env_vars: {
     key: "V2_STAGING_BUCKET"
     # Push non-cloud library docs to `docs-staging-v2-staging` instead of the
     # Cloud RAD bucket `docs-staging-v2`
     value: "docs-staging-v2-staging"
 }
 
 # It will upload the docker image after successful builds.
 env_vars: {
     key: "TRAMPOLINE_IMAGE_UPLOAD"
     value: "true"
 }
 
 # It will always build the docker image.
 env_vars: {
     key: "TRAMPOLINE_DOCKERFILE"
     value: ".kokoro/docker/docs/Dockerfile"
 }
 
 # Fetch the token needed for reporting release status to GitHub
 before_action {
   fetch_keystore {
     keystore_resource {
       keystore_config_id: 73713
       keyname: "yoshi-automation-github-key"
     }
   }
 }
 
 before_action {
   fetch_keystore {
     keystore_resource {
       keystore_config_id: 73713
       keyname: "docuploader_service_account"
     }
   }
 }
diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg
@@ -0,0  1,28 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
 env_vars: {
     key: "STAGING_BUCKET"
     value: "gcloud-python-test"
 }
 
 env_vars: {
     key: "V2_STAGING_BUCKET"
     value: "gcloud-python-test"
 }
 
 # We only upload the image in the main `docs` build.
 env_vars: {
     key: "TRAMPOLINE_IMAGE_UPLOAD"
     value: "false"
 }
 
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/langchain-google-cloud-sql-mssql-python/.kokoro/build.sh"
 }
 
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"
     value: "docs docfx"
 }
diff --git a/.kokoro/docs/docs.cfg b/.kokoro/docs/docs.cfg
@@ -0,0  1 @@
 # Format: //devtools/kokoro/config/proto/build.proto
diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh
@@ -0,0  1,43 @@
 #!/bin/bash
 # Copyright 2024 Google LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 set -eo pipefail
 
 function now { date  "%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
 function msg { println "$*" >&2 ;}
 function println { printf '%s\n' "$(now) $*" ;}
 
 
 # Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
 # kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
 SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
 msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
 mkdir -p ${SECRET_LOCATION}
 for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
 do
   msg "Retrieving secret ${key}"
   docker run --entrypoint=gcloud \
     --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
     gcr.io/google.com/cloudsdktool/cloud-sdk \
     secrets versions access latest \
     --project cloud-devrel-kokoro-resources \
     --secret ${key} > \
     "${SECRET_LOCATION}/${key}"
   if [[ $? == 0 ]]; then
     msg "Secret written to ${SECRET_LOCATION}/${key}"
   else
     msg "Error retrieving secret ${key}"
   fi
 done
diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh
@@ -0,0  1,64 @@
 #!/bin/bash
 # Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 set -eo pipefail
 
 # Disable buffering, so that the logs stream through.
 export PYTHONUNBUFFERED=1
 
 export PATH="${HOME}/.local/bin:${PATH}"
 
 # Install nox
 python3 -m pip install --require-hashes -r .kokoro/requirements.txt
 python3 -m nox --version
 
 # build docs
 nox -s docs
 
 VERSION=$(python -c "import langchain_google_cloud_sql_mssql;print(langchain_google_cloud_sql_mssql.__version__)")
 
 # create metadata
 python3 -m docuploader create-metadata \
   --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
   --version=$VERSION \
   --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
   --distribution-name=$(yq -oy '.project.name'  pyproject.toml) \
   --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
   --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
   --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
 
 cat docs.metadata
 
 # upload docs
 python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
 
 
 # docfx yaml files
 nox -s docfx
 
 # create metadata.
 python3 -m docuploader create-metadata \
   --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
   --version=$VERSION \
   --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
   --distribution-name=$(yq -oy '.project.name'  pyproject.toml) \
   --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
   --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
   --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
 
 cat docs.metadata
 
 # upload docs
 python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"
diff --git a/.kokoro/release.sh b/.kokoro/release.sh
@@ -0,0  1,29 @@
 #!/bin/bash
 # Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 set -eo pipefail
 
 # Start the releasetool reporter
 python3 -m pip install --require-hashes -r github/langchain-google-cloud-sql-mssql-python/.kokoro/requirements.txt
 python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
 
 # Disable buffering, so that the logs stream through.
 export PYTHONUNBUFFERED=1
 
 # Move into the package, build the distribution and upload.
 TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
 cd github/langchain-google-cloud-sql-mssql-python
 python3 -m build --wheel
 twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*