diff --git a/.gitignore b/.gitignore index 0295c98..819327d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ _testmain.go *.test *.prof +vendor/ diff --git a/allthefirmwares.go b/allthefirmwares.go new file mode 100644 index 0000000..89c9f12 --- /dev/null +++ b/allthefirmwares.go @@ -0,0 +1,279 @@ +package main + +import ( + "bytes" + "crypto/sha1" + _ "crypto/sha512" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "text/template" + + "github.com/cheggaaa/pb" + "github.com/dustin/go-humanize" +) + +var ( + // flags + verifyIntegrity, reDownloadOnVerificationFailed, downloadSigned bool + downloadDirectoryTemplate, specifiedDevice string + + // counters + downloadedSize, totalFirmwareSize uint64 + totalFirmwareCount, totalDeviceCount int +) + +func init() { + flag.BoolVar(&verifyIntegrity, "c", false, "just check the integrity of the currently downloaded files (if any)") + flag.BoolVar(&reDownloadOnVerificationFailed, "r", false, "redownload the file if it fails verification (w/ -c)") + flag.BoolVar(&downloadSigned, "s", false, "only download signed firmwares") + flag.StringVar(&downloadDirectoryTemplate, "d", "./", "the location to save/check IPSW files.\n\tCan include templates e.g. {{.Identifier}} or {{.BuildID}}") + flag.StringVar(&specifiedDevice, "i", "", "only download for the specified device") + flag.Parse() +} + +func main() { + // catch interrupt + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + go func() { + for range c { + // sig is a ^C, handle it + fmt.Println() + log.Printf("Downloaded %v\n", humanize.Bytes(uint64(downloadedSize))) + + os.Exit(0) + } + }() + + body, err := getFirmwaresJSON() + + if err != nil { + log.Fatalf("Unable to retrieve firmware information, err: %s", err) + } + + for identifier, device := range body.Devices { + if specifiedDevice != "" && identifier != specifiedDevice { + continue + } + + totalDeviceCount++ + + for _, ipsw := range device.Firmwares { + if downloadSigned && !ipsw.Signed { + continue + } + + directory, err := parseDownloadDirectory(ipsw, identifier) + + if err != nil { + log.Printf("Unable to parse download directory, err: %s", err) + continue + } + + downloadPath := filepath.Join(directory, ipsw.Filename) + + if _, err := os.Stat(downloadPath); os.IsNotExist(err) { + totalFirmwareCount++ + totalFirmwareSize += ipsw.Size + } + } + } + + if !verifyIntegrity { + log.Printf("Downloading: %v IPSW files for %v device(s) (%v)", totalFirmwareCount, totalDeviceCount, humanize.Bytes(totalFirmwareSize)) + } + + for identifier, device := range body.Devices { + if specifiedDevice != "" && identifier != specifiedDevice { + continue + } + + if !verifyIntegrity { + log.Printf("Downloading %d firmwares for %s", len(device.Firmwares), device.Name) + } + + for _, ipsw := range device.Firmwares { + if downloadSigned && !ipsw.Signed { + continue + } + + directory, err := parseDownloadDirectory(ipsw, identifier) + + if err != nil { + log.Printf("Unable to parse download directory, err: %s", err) + continue + } + + // ensure download directory exists + if !verifyIntegrity { + err := os.MkdirAll(directory, 0700) + + if err != nil { + log.Printf("Unable to create download directory: %s, err: %s", directory, err) + break + } + } + + downloadPath := filepath.Join(directory, ipsw.Filename) + + _, err = os.Stat(downloadPath) + + if os.IsNotExist(err) && !verifyIntegrity { + for { + err := downloadWithProgressBar(ipsw, downloadPath) + + if err == nil || !reDownloadOnVerificationFailed { + break + } + } + } else if err == nil && verifyIntegrity { + fileOK, err := verify(downloadPath, ipsw.SHA1) + + if err != nil { + log.Printf("Error verifying: %s, err: %s", ipsw.Filename, err) + } + + if fileOK { + log.Printf("%s verified successfully", ipsw.Filename) + continue + } + + log.Printf("%s did not verify successfully", ipsw.Filename) + + if reDownloadOnVerificationFailed { + for { + err := downloadWithProgressBar(ipsw, downloadPath) + + if err == nil { + break + } + } + } + } else if err != nil && !os.IsNotExist(err) { + log.Printf("Error reading download path: %s, err: %s", downloadPath, err) + } + } + } +} + +func downloadWithProgressBar(ipsw *Firmware, downloadPath string) error { + log.Printf("Downloading %s (%s)", ipsw.Filename, humanize.Bytes(ipsw.Size)) + + bar := pb.New(int(ipsw.Size)).SetUnits(pb.U_BYTES) + bar.Start() + + checksum, err := download(ipsw.URL, downloadPath, bar, func(n, downloaded int, total int64) { + downloadedSize += uint64(n) + }) + + bar.Finish() + + if err != nil { + log.Printf("Error while downloading %s, err: %s", ipsw.Filename, err) + return err + } else if checksum != ipsw.SHA1 { + log.Printf("File: %s failed checksum (wanted: %s, got: %s)", ipsw.Filename, ipsw.SHA1, checksum) + return errors.New("checksum incorrect") + } + + return nil +} + +func parseDownloadDirectory(fw *Firmware, identifier string) (string, error) { + directoryBuffer := new(bytes.Buffer) + + t, err := template.New("firmware").Parse(downloadDirectoryTemplate) + + if err != nil { + return "", err + } + + // add the identifier, for simplicity + fw.Identifier = identifier + + err = t.Execute(directoryBuffer, fw) + + if err != nil { + return "", nil + } + + return directoryBuffer.String(), err +} + +func verify(location string, expectedSHA1sum string) (bool, error) { + file, err := os.Open(location) + + if err != nil { + return false, err + } + + defer file.Close() + + h := sha1.New() + + _, err = io.Copy(h, file) + + if err != nil { + return false, err + } + + bs := h.Sum(nil) + + return expectedSHA1sum == hex.EncodeToString(bs), nil +} + +func download(url string, location string, writer io.Writer, callback func(n, downloaded int, total int64)) (string, error) { + out, err := os.Create(location) + + if err != nil { + return "", err + } + + defer out.Close() + + h := sha1.New() + mw := io.MultiWriter(out, h, writer) + + resp, err := http.Get(url) + + if err != nil { + return "", err + } + + defer resp.Body.Close() + + buf := make([]byte, 128*1024) + + downloaded := 0 + + for { + if n, err := resp.Body.Read(buf); (err == nil || err == io.EOF) && n > 0 { + _, err = mw.Write(buf[:n]) + + if err != nil { + return "", err + } + + downloaded += n + + if callback != nil { + callback(n, downloaded, resp.ContentLength) + } + } else if err != nil && err != io.EOF { + return "", err + } else { + break + } + } + + return hex.EncodeToString(h.Sum(nil)), err +} diff --git a/api.go b/api.go new file mode 100644 index 0000000..81561c8 --- /dev/null +++ b/api.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "net/http" +) + +const ( + apiURL = "https://api.ipsw.me/v3/firmwares.json" + userAgent = "allthefirmwares/1.0" +) + +type Device struct { + Name string `json:"name"` + BoardConfig string `json:"BoardConfig"` + Platform string `json:"platform"` + CPID int `json:"cpid"` + BDID int `json:"bdid"` + Firmwares []*Firmware `json:"firmwares"` +} + +type Firmware struct { + Identifier string `json:"identifier,omitempty"` + Version string `json:"version"` + Device string `json:"device,omitempty"` + BuildID string `json:"buildid"` + SHA1 string `json:"sha1sum"` + MD5 string `json:"md5sum"` + Size uint64 `json:"size"` + ReleaseDate string `json:"releasedate,omitempty"` + UploadDate string `json:"uploaddate"` + URL string `json:"url"` + Signed bool `json:"signed"` + Filename string `json:"filename"` +} + +type APIJSON struct { + Devices map[string]*Device `json:"devices"` +} + +// get the JSON from API_URL and parse it +func getFirmwaresJSON() (parsed *APIJSON, err error) { + req, err := http.NewRequest(http.MethodGet, apiURL, nil) + + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", userAgent) + + response, err := http.DefaultClient.Do(req) + + if err != nil { + return nil, err + } + + defer response.Body.Close() + + contents, err := ioutil.ReadAll(response.Body) + + if err != nil { + return nil, err + } + + err = json.Unmarshal(contents, &parsed) + + if err != nil { + return nil, err + } + + return parsed, err +} diff --git a/download.go b/download.go deleted file mode 100644 index 031fa52..0000000 --- a/download.go +++ /dev/null @@ -1,355 +0,0 @@ -package main - -import ( - "bytes" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "os/signal" - "path/filepath" - "runtime" - "text/template" - _ "crypto/sha512" - "golang.org/x/crypto/ssh/terminal" - "github.com/dustin/go-humanize" -) - -const API_URL = "https://api.ipsw.me/v2.1/firmwares.json" - -type Device struct { - Name string `json:"name"` - BoardConfig string `json:"BoardConfig"` - Platform string `json:"platform"` - CPID int `json:"cpid"` - BDID int `json:"bdid"` - Firmwares []*Firmware `json:"firmwares"` -} - -type Firmware struct { - Identifier string `json:"identifier,omitempty"` - Version string `json:"version"` - Device string `json:"device,omitempty"` - BuildID string `json:"buildid"` - SHA1 string `json:"sha1sum"` - MD5 string `json:"md5sum"` - Size int64 `json:"size"` - ReleaseDate string `json:"releasedate,omitempty"` - UploadDate string `json:"uploaddate"` - URL string `json:"url"` - Signed bool `json:"signed"` - Filename string `json:"filename"` -} - -type iTunes struct { - Platform string `json:"platform,omitempty"` - Version string `json:"version"` - UploadDate string `json:"uploaddate"` - URL string `json:"url"` - SixtyFourBitURL string `json:"64biturl,omitempty"` - ReleaseDate string `json:"releasedate,omitempty"` -} -type APIJSON struct { - Devices map[string]*Device `json:"devices"` - ITunes map[string][]*iTunes `json:"iTunes"` -} - -// args! -var justCheck, redownloadIfBroken, downloadSigned bool -var downloadDirectory, downloadDirectoryTempl, currentIPSW, onlyDevice string -var filesizeDownloaded, totalFirmwareSize int64 -var totalFirmwareCount, totalDeviceCount, downloadCount int - -func init() { - // parse the flags - flag.BoolVar(&justCheck, "c", false, "just check the integrity of the currently downloaded files") - flag.BoolVar(&redownloadIfBroken, "r", false, "redownload the file if it fails verification (w/ -c)") - flag.BoolVar(&downloadSigned, "s", false, "only download signed firmwares") - flag.StringVar(&downloadDirectoryTempl, "d", "./", "the location to save/check IPSW files.\n\t Can include templates e.g. {{.Identifier}} or {{.BuildID}}") - flag.StringVar(&onlyDevice, "i", "", "only download for the specified device") - flag.Parse() -} - -// returns a progress bar fitting the terminal width given a progress percentage -func ProgressBar(progress int) (progressBar string) { - - var width int - - if runtime.GOOS == "windows" { - // we'll just assume it's standard terminal width - width = 80 - } else { - width, _, _ = terminal.GetSize(0) - } - - // take off 26 for extra info (e.g. percentage) - width = width - 26 - - // get the current progress - currentProgress := (progress * width) / 100 - - progressBar = "[" - - // fill up progress - for i := 0; i < currentProgress; i++ { - progressBar = progressBar + "=" - } - - progressBar = progressBar + ">" - - // fill the rest with spaces - for i := width; i > currentProgress; i-- { - progressBar = progressBar + " " - } - - // end the progressbar - progressBar = progressBar + "] " + fmt.Sprintf("%3d", progress) + "%%" - - return progressBar -} - -// get the JSON from API_URL and parse it -func GetFirmwaresJSON() (parsed *APIJSON, err error) { - response, err := http.Get(API_URL) - if err != nil { - return nil, err - } - - file, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - - err = json.Unmarshal(file, &parsed) - - if err != nil { - return nil, err - } - - return parsed, err -} - -// generate a SHA1 of the file, then compare it to a known one -func (fw *Firmware) VerifyFile() (result bool, err error) { - path := filepath.Join(downloadDirectory, fw.Filename) - file, err := os.Open(path) - if err != nil { - return false, err - } - - defer file.Close() - - h := sha1.New() - _, err = io.Copy(h, file) - if err != nil { - return false, err - } - - bs := h.Sum(nil) - - return fw.SHA1 == hex.EncodeToString(bs), nil -} - -// Download the given firmware -func (fw *Firmware) Download() (sha1sum string, err error) { - - //fmt.Println("Downloading to " + filepath.Join(downloadDirectory, filename) + "... ") - - downloadCount++ - - out, err := os.Create(filepath.Join(downloadDirectory, fw.Filename)) - defer out.Close() - - h := sha1.New() - mw := io.MultiWriter(out, h) - - if err != nil { - return "", err - } - - resp, err := http.Get(fw.URL) - if err != nil { - return "", err - } - - defer resp.Body.Close() - - doneCh := make(chan struct{}) - go func() { - size := resp.ContentLength - downloaded := int64(0) - buf := make([]byte, 128*1024) - - for { - if n, _ := resp.Body.Read(buf); n > 0 { - mw.Write(buf[:n]) - downloaded += int64(n) - filesizeDownloaded += int64(n) - pct := int((downloaded * 100) / size) - - fmt.Printf("\r(%d/%d) "+ProgressBar(pct)+" of %4v", downloadCount, totalFirmwareCount, humanize.Bytes(uint64(size))) - } else { - break - } - } - - doneCh <- struct{}{} - }() - <-doneCh - - return hex.EncodeToString(h.Sum(nil)), err -} - -// Read the directory to download from the args, parsing templates if they exist -func (fw *Firmware) ParseDownloadDir(identifier string, makeDir bool) { - - directoryString := new(bytes.Buffer) - - tmpl, err := template.New("firmware").Parse(downloadDirectoryTempl) - - // add the identifier, for simplicity - fw.Identifier = identifier - - if err != nil { - panic(err) - } - - err = tmpl.Execute(directoryString, fw) - - if err != nil { - panic(err) - } - - downloadDirectory = directoryString.String() - - // make the directory - if makeDir { - os.MkdirAll(downloadDirectory, 0700) - } -} - -func main() { - - // so we can catch interrupt - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - - go func() { - for _ = range c { - fmt.Println() - - fmt.Printf("Downloaded %v\n", humanize.Bytes(uint64(filesizeDownloaded))) - - os.Exit(0) - } - }() - - result, err := GetFirmwaresJSON() - - if err != nil { - panic(err) - } - - if !justCheck { - - // calculate the size of the download - - for identifier, info := range result.Devices { - if onlyDevice != "" && identifier != onlyDevice { - continue - } - - totalDeviceCount++ - for _, firmware := range info.Firmwares { - - // don't download unsigned firmwares if we just want signed ones - if downloadSigned && !firmware.Signed { - continue - } - - firmware.ParseDownloadDir(identifier, false) - - if _, err := os.Stat(filepath.Join(downloadDirectory, firmware.Filename)); os.IsNotExist(err) { - totalFirmwareCount++ - totalFirmwareSize += firmware.Size - } - } - } - - fmt.Printf("Downloading %v IPSW files for %v device(s) (%v)\n", totalFirmwareCount, totalDeviceCount, humanize.Bytes(uint64(totalFirmwareSize))) - } - - for identifier, deviceinfo := range result.Devices { - - if onlyDevice != "" && identifier != onlyDevice { - continue - } - - fmt.Printf("\nDevice: %s (%s) - %v firmwares\n", deviceinfo.Name, identifier, len(deviceinfo.Firmwares)) - fmt.Println("------------------------------------------------------\n") - - for _, firmware := range deviceinfo.Firmwares { - - firmware.ParseDownloadDir(identifier, !justCheck) - - // don't download unsigned firmwares if we just want signed ones - if downloadSigned && !firmware.Signed { - continue - } - - fmt.Print("Checking if " + firmware.Filename + " exists... ") - if _, err := os.Stat(filepath.Join(downloadDirectory, firmware.Filename)); os.IsNotExist(err) && !justCheck { - - fmt.Println("needs downloading") - shasum, err := firmware.Download() - - if err != nil { - fmt.Println(err) - } else { - - // not sure if these will display properly - if shasum == firmware.SHA1 { - fmt.Println("✔") - } else { - fmt.Println("✘") - } - } - - } else if _, err := os.Stat(filepath.Join(downloadDirectory, firmware.Filename)); !os.IsNotExist(err) && justCheck { - fmt.Println("true") - - fmt.Print("\tchecking file... ") - - if fileOK, _ := firmware.VerifyFile(); fileOK { - fmt.Println("✔ ok") - } else { - fmt.Println("✘ bad") - if redownloadIfBroken { - fmt.Println("Redownloading...") - shasum, err := firmware.Download() - - if err != nil { - fmt.Println(err) - } - - if shasum == firmware.SHA1 { - fmt.Println("✔") - } else { - fmt.Println("✘") - } - } - } - - } else { - fmt.Println("false") - } - } - } - - fmt.Printf("Downloaded %v\n", humanize.Bytes(uint64(filesizeDownloaded))) -} diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..0a5130c --- /dev/null +++ b/glide.lock @@ -0,0 +1,18 @@ +hash: e77709891038cfae93eeb58c8d90538a3b98f0d7857e4d40d44e7acffa1efe83 +updated: 2017-04-25T20:19:15.383155278+01:00 +imports: +- name: github.com/cheggaaa/pb + version: b6229822fa186496fcbf34111237e7a9693c6971 +- name: github.com/dustin/go-humanize + version: 259d2a102b871d17f30e3cd9881a642961a1e486 +- name: github.com/mattn/go-runewidth + version: 14207d285c6c197daabb5c9793d63e7af9ab2d50 +- name: golang.org/x/crypto + version: c2303dcbe84172e0c0da4c9f083eeca54c06f298 + subpackages: + - ssh/terminal +- name: golang.org/x/sys + version: afadfcc7779c1f4db0f6f6438afcb108d9c9c7cd + subpackages: + - unix +testImports: [] diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..da0c737 --- /dev/null +++ b/glide.yaml @@ -0,0 +1,8 @@ +package: github.com/cj123/allthefirmwares +import: +- package: github.com/cheggaaa/pb + version: ^1.0.13 +- package: github.com/dustin/go-humanize +- package: golang.org/x/crypto + subpackages: + - ssh/terminal