1*9e965d6fSRomain Jobredeaux// Copyright 2018 The Bazel Authors. All rights reserved. 2*9e965d6fSRomain Jobredeaux// 3*9e965d6fSRomain Jobredeaux// Licensed under the Apache License, Version 2.0 (the "License"); 4*9e965d6fSRomain Jobredeaux// you may not use this file except in compliance with the License. 5*9e965d6fSRomain Jobredeaux// You may obtain a copy of the License at 6*9e965d6fSRomain Jobredeaux// 7*9e965d6fSRomain Jobredeaux// http://www.apache.org/licenses/LICENSE-2.0 8*9e965d6fSRomain Jobredeaux// 9*9e965d6fSRomain Jobredeaux// Unless required by applicable law or agreed to in writing, software 10*9e965d6fSRomain Jobredeaux// distributed under the License is distributed on an "AS IS" BASIS, 11*9e965d6fSRomain Jobredeaux// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12*9e965d6fSRomain Jobredeaux// See the License for the specific language governing permissions and 13*9e965d6fSRomain Jobredeaux// limitations under the License. 14*9e965d6fSRomain Jobredeaux 15*9e965d6fSRomain Jobredeaux// Package ziputils provides utility functions to work with zip files. 16*9e965d6fSRomain Jobredeauxpackage ziputils 17*9e965d6fSRomain Jobredeaux 18*9e965d6fSRomain Jobredeauximport ( 19*9e965d6fSRomain Jobredeaux "archive/zip" 20*9e965d6fSRomain Jobredeaux "bytes" 21*9e965d6fSRomain Jobredeaux "io" 22*9e965d6fSRomain Jobredeaux "io/ioutil" 23*9e965d6fSRomain Jobredeaux "os" 24*9e965d6fSRomain Jobredeaux "path/filepath" 25*9e965d6fSRomain Jobredeaux "strings" 26*9e965d6fSRomain Jobredeaux "time" 27*9e965d6fSRomain Jobredeaux 28*9e965d6fSRomain Jobredeaux "golang.org/x/sync/errgroup" 29*9e965d6fSRomain Jobredeaux) 30*9e965d6fSRomain Jobredeaux 31*9e965d6fSRomain Jobredeaux// Empty file contains only the End of central directory record. 0x06054b50 32*9e965d6fSRomain Jobredeaux// https://en.wikipedia.org/wiki/Zip_(file_format) 33*9e965d6fSRomain Jobredeauxvar ( 34*9e965d6fSRomain Jobredeaux emptyzip = append([]byte{0x50, 0x4b, 0x05, 0x06}, make([]byte, 18)...) 35*9e965d6fSRomain Jobredeaux dirPerm os.FileMode = 0755 36*9e965d6fSRomain Jobredeaux) 37*9e965d6fSRomain Jobredeaux 38*9e965d6fSRomain Jobredeaux// EmptyZipReader wraps an reader whose contents are the empty zip. 39*9e965d6fSRomain Jobredeauxtype EmptyZipReader struct { 40*9e965d6fSRomain Jobredeaux *bytes.Reader 41*9e965d6fSRomain Jobredeaux} 42*9e965d6fSRomain Jobredeaux 43*9e965d6fSRomain Jobredeaux// NewEmptyZipReader creates and returns an EmptyZipReader struct. 44*9e965d6fSRomain Jobredeauxfunc NewEmptyZipReader() *EmptyZipReader { 45*9e965d6fSRomain Jobredeaux return &EmptyZipReader{bytes.NewReader(emptyzip)} 46*9e965d6fSRomain Jobredeaux} 47*9e965d6fSRomain Jobredeaux 48*9e965d6fSRomain Jobredeaux// EmptyZip creates empty zip archive. 49*9e965d6fSRomain Jobredeauxfunc EmptyZip(dst string) error { 50*9e965d6fSRomain Jobredeaux zipfile, err := os.Create(dst) 51*9e965d6fSRomain Jobredeaux if err != nil { 52*9e965d6fSRomain Jobredeaux return err 53*9e965d6fSRomain Jobredeaux } 54*9e965d6fSRomain Jobredeaux defer zipfile.Close() 55*9e965d6fSRomain Jobredeaux _, err = io.Copy(zipfile, NewEmptyZipReader()) 56*9e965d6fSRomain Jobredeaux return err 57*9e965d6fSRomain Jobredeaux} 58*9e965d6fSRomain Jobredeaux 59*9e965d6fSRomain Jobredeaux// Zip archives src into dst without compression. 60*9e965d6fSRomain Jobredeauxfunc Zip(src, dst string) error { 61*9e965d6fSRomain Jobredeaux fi, err := os.Stat(src) 62*9e965d6fSRomain Jobredeaux if err != nil { 63*9e965d6fSRomain Jobredeaux return err 64*9e965d6fSRomain Jobredeaux } 65*9e965d6fSRomain Jobredeaux 66*9e965d6fSRomain Jobredeaux zipfile, err := os.Create(dst) 67*9e965d6fSRomain Jobredeaux if err != nil { 68*9e965d6fSRomain Jobredeaux return err 69*9e965d6fSRomain Jobredeaux } 70*9e965d6fSRomain Jobredeaux defer zipfile.Close() 71*9e965d6fSRomain Jobredeaux 72*9e965d6fSRomain Jobredeaux archive := zip.NewWriter(zipfile) 73*9e965d6fSRomain Jobredeaux defer archive.Close() 74*9e965d6fSRomain Jobredeaux 75*9e965d6fSRomain Jobredeaux if !fi.Mode().IsDir() { 76*9e965d6fSRomain Jobredeaux return WriteFile(archive, src, filepath.Base(src)) 77*9e965d6fSRomain Jobredeaux } 78*9e965d6fSRomain Jobredeaux 79*9e965d6fSRomain Jobredeaux return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { 80*9e965d6fSRomain Jobredeaux if err != nil { 81*9e965d6fSRomain Jobredeaux return err 82*9e965d6fSRomain Jobredeaux } 83*9e965d6fSRomain Jobredeaux 84*9e965d6fSRomain Jobredeaux if info.IsDir() { 85*9e965d6fSRomain Jobredeaux return nil 86*9e965d6fSRomain Jobredeaux } 87*9e965d6fSRomain Jobredeaux 88*9e965d6fSRomain Jobredeaux return WriteFile(archive, path, strings.TrimPrefix(path, src+string(filepath.Separator))) 89*9e965d6fSRomain Jobredeaux }) 90*9e965d6fSRomain Jobredeaux} 91*9e965d6fSRomain Jobredeaux 92*9e965d6fSRomain Jobredeaux// WriteFile writes filename to the out zip writer. 93*9e965d6fSRomain Jobredeauxfunc WriteFile(out *zip.Writer, filename, zipFilename string) error { 94*9e965d6fSRomain Jobredeaux // It's important to set timestamps to zero, otherwise we would break caching for unchanged files 95*9e965d6fSRomain Jobredeaux f, err := out.CreateHeader(&zip.FileHeader{Name: zipFilename, Method: zip.Store, Modified: time.Unix(0, 0)}) 96*9e965d6fSRomain Jobredeaux if err != nil { 97*9e965d6fSRomain Jobredeaux return err 98*9e965d6fSRomain Jobredeaux } 99*9e965d6fSRomain Jobredeaux contents, err := ioutil.ReadFile(filename) 100*9e965d6fSRomain Jobredeaux if err != nil { 101*9e965d6fSRomain Jobredeaux return err 102*9e965d6fSRomain Jobredeaux } 103*9e965d6fSRomain Jobredeaux _, err = f.Write(contents) 104*9e965d6fSRomain Jobredeaux return err 105*9e965d6fSRomain Jobredeaux} 106*9e965d6fSRomain Jobredeaux 107*9e965d6fSRomain Jobredeaux// WriteReader writes a reader to the out zip writer. 108*9e965d6fSRomain Jobredeauxfunc WriteReader(out *zip.Writer, in io.Reader, filename string) error { 109*9e965d6fSRomain Jobredeaux // It's important to set timestamps to zero, otherwise we would break caching for unchanged files 110*9e965d6fSRomain Jobredeaux f, err := out.CreateHeader(&zip.FileHeader{Name: filename, Method: zip.Store, Modified: time.Unix(0, 0)}) 111*9e965d6fSRomain Jobredeaux if err != nil { 112*9e965d6fSRomain Jobredeaux return err 113*9e965d6fSRomain Jobredeaux } 114*9e965d6fSRomain Jobredeaux contents, err := ioutil.ReadAll(in) 115*9e965d6fSRomain Jobredeaux if err != nil { 116*9e965d6fSRomain Jobredeaux return err 117*9e965d6fSRomain Jobredeaux } 118*9e965d6fSRomain Jobredeaux _, err = f.Write(contents) 119*9e965d6fSRomain Jobredeaux return err 120*9e965d6fSRomain Jobredeaux} 121*9e965d6fSRomain Jobredeaux 122*9e965d6fSRomain Jobredeaux// Unzip expands srcZip in dst directory 123*9e965d6fSRomain Jobredeauxfunc Unzip(srcZip, dst string) error { 124*9e965d6fSRomain Jobredeaux reader, err := zip.OpenReader(srcZip) 125*9e965d6fSRomain Jobredeaux if err != nil { 126*9e965d6fSRomain Jobredeaux return err 127*9e965d6fSRomain Jobredeaux } 128*9e965d6fSRomain Jobredeaux defer reader.Close() 129*9e965d6fSRomain Jobredeaux 130*9e965d6fSRomain Jobredeaux _, err = os.Stat(dst) 131*9e965d6fSRomain Jobredeaux if err != nil && !os.IsNotExist(err) { 132*9e965d6fSRomain Jobredeaux return err 133*9e965d6fSRomain Jobredeaux } 134*9e965d6fSRomain Jobredeaux if os.IsNotExist(err) { 135*9e965d6fSRomain Jobredeaux if err := os.MkdirAll(dst, dirPerm); err != nil { 136*9e965d6fSRomain Jobredeaux return err 137*9e965d6fSRomain Jobredeaux } 138*9e965d6fSRomain Jobredeaux } 139*9e965d6fSRomain Jobredeaux 140*9e965d6fSRomain Jobredeaux for _, file := range reader.File { 141*9e965d6fSRomain Jobredeaux path := filepath.Join(dst, file.Name) 142*9e965d6fSRomain Jobredeaux 143*9e965d6fSRomain Jobredeaux if file.FileInfo().IsDir() { 144*9e965d6fSRomain Jobredeaux if err := os.MkdirAll(path, dirPerm); err != nil { 145*9e965d6fSRomain Jobredeaux return err 146*9e965d6fSRomain Jobredeaux } 147*9e965d6fSRomain Jobredeaux continue 148*9e965d6fSRomain Jobredeaux } 149*9e965d6fSRomain Jobredeaux 150*9e965d6fSRomain Jobredeaux dir := filepath.Dir(path) 151*9e965d6fSRomain Jobredeaux _, err := os.Stat(dir) 152*9e965d6fSRomain Jobredeaux if err != nil && !os.IsNotExist(err) { 153*9e965d6fSRomain Jobredeaux return err 154*9e965d6fSRomain Jobredeaux } 155*9e965d6fSRomain Jobredeaux if os.IsNotExist(err) { 156*9e965d6fSRomain Jobredeaux if err := os.MkdirAll(dir, dirPerm); err != nil { 157*9e965d6fSRomain Jobredeaux return err 158*9e965d6fSRomain Jobredeaux } 159*9e965d6fSRomain Jobredeaux } 160*9e965d6fSRomain Jobredeaux 161*9e965d6fSRomain Jobredeaux if err := write(file, path); err != nil { 162*9e965d6fSRomain Jobredeaux return err 163*9e965d6fSRomain Jobredeaux } 164*9e965d6fSRomain Jobredeaux } 165*9e965d6fSRomain Jobredeaux 166*9e965d6fSRomain Jobredeaux return nil 167*9e965d6fSRomain Jobredeaux} 168*9e965d6fSRomain Jobredeaux 169*9e965d6fSRomain Jobredeaux// UnzipParallel expands zip archives in parallel. 170*9e965d6fSRomain Jobredeaux// TODO(b/137549283) Update UnzipParallel and add test 171*9e965d6fSRomain Jobredeauxfunc UnzipParallel(srcZipDestMap map[string]string) error { 172*9e965d6fSRomain Jobredeaux var eg errgroup.Group 173*9e965d6fSRomain Jobredeaux for z, d := range srcZipDestMap { 174*9e965d6fSRomain Jobredeaux zip, dest := z, d 175*9e965d6fSRomain Jobredeaux eg.Go(func() error { return Unzip(zip, dest) }) 176*9e965d6fSRomain Jobredeaux } 177*9e965d6fSRomain Jobredeaux return eg.Wait() 178*9e965d6fSRomain Jobredeaux} 179*9e965d6fSRomain Jobredeaux 180*9e965d6fSRomain Jobredeauxfunc write(zf *zip.File, path string) error { 181*9e965d6fSRomain Jobredeaux rc, err := zf.Open() 182*9e965d6fSRomain Jobredeaux if err != nil { 183*9e965d6fSRomain Jobredeaux return err 184*9e965d6fSRomain Jobredeaux } 185*9e965d6fSRomain Jobredeaux defer rc.Close() 186*9e965d6fSRomain Jobredeaux f, err := os.Create(path) 187*9e965d6fSRomain Jobredeaux if err != nil { 188*9e965d6fSRomain Jobredeaux return err 189*9e965d6fSRomain Jobredeaux } 190*9e965d6fSRomain Jobredeaux defer f.Close() 191*9e965d6fSRomain Jobredeaux _, err = io.Copy(f, rc) 192*9e965d6fSRomain Jobredeaux return err 193*9e965d6fSRomain Jobredeaux} 194