From: Dan Brown Date: Sun, 10 Apr 2022 15:49:21 +0000 (+0100) Subject: Added go-export-page-content example X-Git-Url: http://source.bookstackapp.com/api-scripts/commitdiff_plain/29922fcfeb9a13bf59c9c0a4ebda7d57865aa3ef Added go-export-page-content example --- diff --git a/go-export-page-content/.gitignore b/go-export-page-content/.gitignore new file mode 100644 index 0000000..0674298 --- /dev/null +++ b/go-export-page-content/.gitignore @@ -0,0 +1,6 @@ +bookstack-export/ +page-export/ +bookstack-export +bookstack-export.exe +.idea/ +bin/ diff --git a/go-export-page-content/api.go b/go-export-page-content/api.go new file mode 100644 index 0000000..6e5e761 --- /dev/null +++ b/go-export-page-content/api.go @@ -0,0 +1,140 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" +) + +type BookStackApi struct { + BaseURL string + TokenID string + TokenSecret string +} + +func NewBookStackApi(baseUrl string, tokenId string, tokenSecret string) *BookStackApi { + api := &BookStackApi{ + BaseURL: baseUrl, + TokenID: tokenId, + TokenSecret: tokenSecret, + } + + return api +} + +func (bs BookStackApi) authHeader() string { + return fmt.Sprintf("Token %s:%s", bs.TokenID, bs.TokenSecret) +} + +func (bs BookStackApi) getRequest(method string, urlPath string, data map[string]string) *http.Request { + method = strings.ToUpper(method) + completeUrlStr := fmt.Sprintf("%s/api/%s", strings.TrimRight(bs.BaseURL, "/"), strings.TrimLeft(urlPath, "/")) + + queryValues := url.Values{} + for k, v := range data { + queryValues.Add(k, v) + } + encodedData := queryValues.Encode() + + r, err := http.NewRequest(method, completeUrlStr, strings.NewReader(encodedData)) + if err != nil { + panic(err) + } + + r.Header.Add("Authorization", bs.authHeader()) + + if method != "GET" && method != "HEAD" { + r.Header.Add("Content-Type", "application/x-www-form-urlencoded") + r.Header.Add("Content-Length", strconv.Itoa(len(encodedData))) + } else { + r.URL.RawQuery = encodedData + } + + return r +} + +func (bs BookStackApi) doRequest(method string, urlPath string, data map[string]string) []byte { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + r := bs.getRequest(method, urlPath, data) + res, err := client.Do(r) + if err != nil { + panic(err) + } + + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + panic(err) + } + + return body +} + +func (bs BookStackApi) getFromListResponse(responseData []byte, models any) ListResponse { + var response ListResponse + + if err := json.Unmarshal(responseData, &response); err != nil { + panic(err) + } + + if err := json.Unmarshal(response.Data, models); err != nil { + panic(err) + } + + return response +} + +func (bs BookStackApi) GetBooks(count int, page int) ([]Book, int) { + var books []Book + + data := bs.doRequest("GET", "/books", getPagingParams(count, page)) + response := bs.getFromListResponse(data, &books) + + return books, response.Total +} + +func (bs BookStackApi) GetChapters(count int, page int) ([]Chapter, int) { + var chapters []Chapter + + data := bs.doRequest("GET", "/chapters", getPagingParams(count, page)) + response := bs.getFromListResponse(data, &chapters) + + return chapters, response.Total +} + +func (bs BookStackApi) GetPages(count int, page int) ([]Page, int) { + var pages []Page + + data := bs.doRequest("GET", "/pages", getPagingParams(count, page)) + response := bs.getFromListResponse(data, &pages) + + return pages, response.Total +} + +func (bs BookStackApi) GetPage(id int) Page { + var page Page + + data := bs.doRequest("GET", fmt.Sprintf("/pages/%d", id), nil) + if err := json.Unmarshal(data, &page); err != nil { + panic(err) + } + + return page +} + +func getPagingParams(count int, page int) map[string]string { + return map[string]string{ + "count": strconv.Itoa(count), + "offset": strconv.Itoa(count * (page - 1)), + } +} diff --git a/go-export-page-content/build.sh b/go-export-page-content/build.sh new file mode 100755 index 0000000..0b0cdf2 --- /dev/null +++ b/go-export-page-content/build.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +GOOS=windows GOARCH=amd64 go build -ldflags "-s -w" -o bin/bookstack-export.exe +GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o bin/bookstack-export +GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w" -o bin/bookstack-export-macos + +upx bin/* \ No newline at end of file diff --git a/go-export-page-content/content-map-funcs.go b/go-export-page-content/content-map-funcs.go new file mode 100644 index 0000000..f885789 --- /dev/null +++ b/go-export-page-content/content-map-funcs.go @@ -0,0 +1,68 @@ +package main + +import ( + "time" +) + +func getBookMap(api *BookStackApi) map[int]Book { + var books []Book + var byId = make(map[int]Book) + + page := 1 + hasMoreBooks := true + for hasMoreBooks { + time.Sleep(time.Second / 2) + newBooks, _ := api.GetBooks(200, page) + hasMoreBooks = len(newBooks) == 200 + page++ + books = append(books, newBooks...) + } + + for _, book := range books { + byId[book.Id] = book + } + + return byId +} + +func getChapterMap(api *BookStackApi) map[int]Chapter { + var chapters []Chapter + var byId = make(map[int]Chapter) + + page := 1 + hasMoreChapters := true + for hasMoreChapters { + time.Sleep(time.Second / 2) + newChapters, _ := api.GetChapters(200, page) + hasMoreChapters = len(newChapters) == 200 + page++ + chapters = append(chapters, newChapters...) + } + + for _, chapter := range chapters { + byId[chapter.Id] = chapter + } + + return byId +} + +func getPageMap(api *BookStackApi) map[int]Page { + var pages []Page + var byId = make(map[int]Page) + + page := 1 + hasMorePages := true + for hasMorePages { + time.Sleep(time.Second / 2) + newPages, _ := api.GetPages(200, page) + hasMorePages = len(newPages) == 200 + page++ + pages = append(pages, newPages...) + } + + for _, page := range pages { + byId[page.Id] = page + } + + return byId +} diff --git a/go-export-page-content/export.go b/go-export-page-content/export.go new file mode 100644 index 0000000..8825bf6 --- /dev/null +++ b/go-export-page-content/export.go @@ -0,0 +1,87 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "time" +) + +func main() { + + baseUrlPtr := flag.String("baseurl", "", "The base URL of your BookStack instance") + tokenId := flag.String("tokenid", "", "Your BookStack API Token ID") + tokenSecret := flag.String("tokensecret", "", "Your BookStack API Token Secret") + exportDir := flag.String("exportdir", "./page-export", "The directory to store exported data") + + flag.Parse() + + if *baseUrlPtr == "" || *tokenId == "" || *tokenSecret == "" { + panic("baseurl, tokenid and tokensecret arguments are required") + } + + api := NewBookStackApi(*baseUrlPtr, *tokenId, *tokenSecret) + + // Grab all content from BookStack + fmt.Println("Fetching books...") + bookIdMap := getBookMap(api) + fmt.Printf("Fetched %d books\n", len(bookIdMap)) + fmt.Println("Fetching chapters...") + chapterIdMap := getChapterMap(api) + fmt.Printf("Fetched %d chapters\n", len(chapterIdMap)) + fmt.Println("Fetching pages...") + pageIdMap := getPageMap(api) + fmt.Printf("Fetched %d pages\n", len(pageIdMap)) + + // Track progress when going through our pages + pageCount := len(pageIdMap) + currentCount := 1 + + // Cycle through each of our fetches pages + for _, p := range pageIdMap { + fmt.Printf("Exporting page %d/%d [%s]\n", currentCount, pageCount, p.Name) + // Get the full page content + fullPage := api.GetPage(p.Id) + + // Work out a book+chapter relative path + book := bookIdMap[fullPage.BookId] + path := book.Slug + if chapter, ok := chapterIdMap[fullPage.ChapterId]; ok { + path = "/" + chapter.Slug + } + + // Get the html, or markdown, content from our page along with the file name + // based upon the page slug + content := fullPage.Html + fName := fullPage.Slug + ".html" + if fullPage.Markdown != "" { + content = fullPage.Markdown + fName = fullPage.Slug + ".md" + } + + // Create our directory path + absExportPath, err := filepath.Abs(*exportDir) + if err != nil { + panic(err) + } + + absPath := filepath.Join(absExportPath, path) + err = os.MkdirAll(absPath, 0744) + if err != nil { + panic(err) + } + + // Write the content to the filesystem + fPath := filepath.Join(absPath, fName) + err = os.WriteFile(fPath, []byte(content), 0644) + if err != nil { + panic(err) + } + + // Wait to avoid hitting rate limits + time.Sleep(time.Second / 4) + currentCount++ + } + +} diff --git a/go-export-page-content/go.mod b/go-export-page-content/go.mod new file mode 100644 index 0000000..24d2e67 --- /dev/null +++ b/go-export-page-content/go.mod @@ -0,0 +1,3 @@ +module bookstack-export + +go 1.18 diff --git a/go-export-page-content/models.go b/go-export-page-content/models.go new file mode 100644 index 0000000..426d686 --- /dev/null +++ b/go-export-page-content/models.go @@ -0,0 +1,54 @@ +package main + +import ( + "encoding/json" + "time" +) + +type ListResponse struct { + Data json.RawMessage `json:"data"` + Total int `json:"total"` +} + +type Book struct { + Id int `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` + Description string `json:"description"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + CreatedBy int `json:"created_by"` + UpdatedBy int `json:"updated_by"` + OwnedBy int `json:"owned_by"` + ImageId int `json:"image_id"` +} + +type Chapter struct { + Id int `json:"id"` + BookId int `json:"book_id"` + Name string `json:"name"` + Slug string `json:"slug"` + Description string `json:"description"` + Priority int `json:"priority"` + CreatedAt string `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + CreatedBy int `json:"created_by"` + UpdatedBy int `json:"updated_by"` + OwnedBy int `json:"owned_by"` +} + +type Page struct { + Id int `json:"id"` + BookId int `json:"book_id"` + ChapterId int `json:"chapter_id"` + Name string `json:"name"` + Slug string `json:"slug"` + Html string `json:"html"` + Priority int `json:"priority"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Draft bool `json:"draft"` + Markdown string `json:"markdown"` + RevisionCount int `json:"revision_count"` + Template bool `json:"template"` +} diff --git a/go-export-page-content/readme.md b/go-export-page-content/readme.md new file mode 100644 index 0000000..9fea82c --- /dev/null +++ b/go-export-page-content/readme.md @@ -0,0 +1,38 @@ +# Export Page Content + +This project, written in Go, will export all page content in its original written form (HTML or Markdown). +Content will be written into a directory structure that mirrors the page's location within the BookStack content hierarchy (Book > Chapter > Page). + +Note: This is only provided as an example. The project lacks full error handling and also disables HTTPS verification for easier use with self-signed certificates. + +## Requirements + +[Go](https://go.dev/) is required to build this project. +This project was built and tested using Go 1.18. + +You will need your BookStack API credentials at the ready. + +## Building + +```bash +# Clone down the api-scripts repo and enter this directory +git clone https://github.com/BookStackApp/api-scripts.git +cd api-scripts/go-export-page-content +go build +``` + +This will output a `bookstack-export` executable file. + +A `build.sh` script is provided to build compressed binaries for multiple platforms. +This requires `upx` for the compression element. + +## Running + +You can run the project by running the executable file like so: + +```bash +./bookstack-export --baseurl=https://bookstack.example.com --tokenid=abc123 --tokensecret=def456 +``` + +By default, this will output to a `page-export` directory within the current working directory. +You can define the output directory via a `--exportdir=` option. \ No newline at end of file