1#!/bin/bash 2# Copyright 2022 Google LLC 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16set -eo pipefail 17 18if [[ -z "${CREDENTIALS}" ]]; then 19 CREDENTIALS=${KOKORO_KEYSTORE_DIR}/73713_docuploader_service_account 20fi 21 22if [[ -z "${STAGING_BUCKET_V2}" ]]; then 23 echo "Need to set STAGING_BUCKET_V2 environment variable" 24 exit 1 25fi 26 27# work from the git root directory 28pushd $(dirname "$0")/../../ 29 30root_dir=$(pwd) 31 32python3 --version 33 34# install docuploader package 35python3 -m pip install --require-hashes -r .kokoro/requirements.txt 36 37# If DOCLET_VERSION is passed in (overriding version in shared-config) 38if [ -n "${DOCLET_VERSION}" ]; then 39 doclet_name="java-docfx-doclet-${DOCLET_VERSION}.jar" 40fi 41 42mvn -B -ntp \ 43 -DtrimStackTrace=false \ 44 -Dclirr.skip=true \ 45 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 46 -Denforcer.skip=true \ 47 -Dcheckstyle.skip=true \ 48 -Dflatten.skip=true \ 49 -Danimal.sniffer.skip=true \ 50 -DskipTests=true \ 51 -Djacoco.skip=true \ 52 -T 1C \ 53 install 54 55if [[ -z "${MODULE_LIST}" ]]; then 56 # Retrieve list of modules from aggregator pom 57 modules=($(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g')) 58else 59 modules=($(echo "${MODULE_LIST}" | tr ',' ' ')) 60fi 61excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-shared-dependencies') 62# TODO: Maps docs exclusion logic to be removed once we move to correct location on devsite. See b/262712184 and b/262600829 63while IFS= read -r -d $'\0'; do 64 excluded_modules+=("$REPLY") 65done < <(find java-maps-* -type d -maxdepth 0 -print0) 66echo "Excluded modules: ${excluded_modules[*]}" 67failed_modules=() 68 69for module in "${modules[@]}"; do 70 # Proceed if module is not excluded 71 # Spaces are intentionally added -- Query is regex and array elements are space separated 72 # It tries to match the *exact* `module` text 73 if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then 74 pushd $module 75 # Extract Cloud RAD module name from `distribution_name` in .repo-metadata.json 76 NAME=$(grep -o '"distribution_name": "[^"]*' .repo-metadata.json | grep -o '[^"]*$' | cut -d ':' -f 2) 77 # Extract (current) version from root `versions.txt` file and remove `-SNAPSHOT` 78 VERSION=$(grep "^${NAME}:" "${root_dir}/versions.txt" | cut -d: -f3 | sed -e 's/-SNAPSHOT//g') 79 echo "Running for ${NAME}-${VERSION}" 80 81 # Cloud RAD generation 82 if [ -z "${doclet_name}" ]; then 83 mvn clean -B -ntp \ 84 -P docFX \ 85 -Dclirr.skip=true \ 86 -Denforcer.skip=true \ 87 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 88 -Dcheckstyle.skip=true \ 89 -Dflatten.skip=true \ 90 -Danimal.sniffer.skip=true \ 91 javadoc:aggregate 92 else 93 mvn clean -B -ntp \ 94 -P docFX \ 95 -DdocletPath=${KOKORO_GFILE_DIR}/${doclet_name} \ 96 -Dclirr.skip=true \ 97 -Denforcer.skip=true \ 98 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 99 -Dcheckstyle.skip=true \ 100 -Dflatten.skip=true \ 101 -Danimal.sniffer.skip=true \ 102 javadoc:aggregate 103 fi 104 105 if [ "$?" -ne "0" ]; then 106 failed_modules+=("${module}") 107 continue 108 fi 109 # include CHANGELOG if exists 110 if [ -e CHANGELOG.md ]; then 111 cp CHANGELOG.md target/docfx-yml/history.md 112 fi 113 pushd target/docfx-yml 114 115 echo "Creating metadata for ${module}..." 116 # create metadata 117 python3 -m docuploader create-metadata \ 118 --name ${NAME} \ 119 --version ${VERSION} \ 120 --xrefs devsite://java/gax \ 121 --xrefs devsite://java/google-cloud-core \ 122 --xrefs devsite://java/api-common \ 123 --xrefs devsite://java/proto-google-common-protos \ 124 --xrefs devsite://java/google-api-client \ 125 --xrefs devsite://java/google-http-client \ 126 --xrefs devsite://java/protobuf \ 127 --language java 128 129 echo "Uploading tarball for ${module}..." 130 # upload yml to production bucket 131 python3 -m docuploader upload . \ 132 --credentials ${CREDENTIALS} \ 133 --staging-bucket ${STAGING_BUCKET_V2} \ 134 --destination-prefix docfx 135 136 echo "Uploaded tarball for ${module}" 137 138 popd # out of target/docfx-yml 139 popd # out of $module 140 fi 141done 142 143if [ ${#failed_modules[@]} -eq 0 ]; then 144 echo "All modules uploaded to CloudRAD" 145else 146 echo "These modules failed: ${failed_modules[*]}" 147 exit 1 148fi 149