Skip to content

Commit

Permalink
Add a route with a list of all cached requests (#22)
Browse files Browse the repository at this point in the history
  • Loading branch information
@man90es
man90es authored Jul 29, 2024
1 parent b56bc19 commit 1571316
Show file tree
Hide file tree
Showing 10 changed files with 213 additions and 7 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 1,4 @@
FROM golang:1.21-alpine AS build
FROM golang:1.22-alpine AS build
RUN apk add --no-cache git
WORKDIR /src/bdo-rest-api
COPY . .
Expand Down
5 changes: 3 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 3,9 @@
A scraper for Black Desert Online player in-game data with a REST API. It currently supports European, North American, and South American servers (Korean server support is in progress).

## Projects using this API
- [BDO Leaderboards](https://bdo.hemlo.cc/leaderboards) ([Source](https://github.com/man90es/BDO-Leaderboards)): web-based leaderboards for Black Desert Online.
- [Ikusa](https://ikusa.site) ([Source](https://github.com/sch-28/ikusa_api)): a powerful tool that allows you to analyze your game logs and gain valuable insights into your combat performance.
- BDO Leaderboards ([Website](https://bdo.hemlo.cc/leaderboards), [sources](https://github.com/man90es/BDO-Leaderboards)): web-based leaderboards for Black Desert Online.
- Ikusa ([Website](https://ikusa.site), [sources](https://github.com/sch-28/ikusa_api)): powerful tool that allows you to analyze your game logs and gain valuable insights into your combat performance.
- GuildYapper ([Discord server](https://discord.gg/x2nKYuu2Z2)): Discord bot with various features for BDO guilds such as guild and player history logging, and automatic trial Discord management (more features TBA).

## How to start using it
There are two ways to use this scraper for your needs:
Expand Down
5 changes: 5 additions & 0 deletions cache/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 5,7 @@ import (
"time"

goCache "github.com/patrickmn/go-cache"
"golang.org/x/exp/maps"

"bdo-rest-api/config"
"bdo-rest-api/utils"
Expand Down Expand Up @@ -64,3 65,7 @@ func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, ex
func (c *cache[T]) GetItemCount() int {
return c.internalCache.ItemCount()
}

func (c *cache[T]) GetKeys() []string {
return maps.Keys(c.internalCache.Items())
}
135 changes: 135 additions & 0 deletions docs/openapi.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 77,7 @@
"type": "number"
},
"/guild/search": {
"deprecated": true,
"type": "number"
}
}
Expand All @@ -96,6 97,10 @@
}
}
},
"docs": {
"type": "string",
"example": "https://man90es.github.io/BDO-REST-API"
},
"proxies": {
"type": "number"
},
Expand Down Expand Up @@ -404,6 409,7 @@
"name": "page",
"in": "query",
"description": "This parameter is understood by the API, but you should either omit it or set to 1. Because of how search currently works, there is never more than one page.",
"deprecated": true,
"schema": {
"type": "number",
"default": 1
Expand Down Expand Up @@ -600,6 606,7 @@
"summary": "Search for a guild.",
"description": "Search for a guild by combination of its region and name.",
"operationId": "getGuildSearch",
"deprecated": true,
"parameters": [
{
"name": "query",
Expand Down Expand Up @@ -691,6 698,134 @@
}
}
}
},
"/v1/cache": {
"get": {
"summary": "Retrieve cached routes",
"operationId": "getCache",
"responses": {
"200": {
"description": "OK.",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"/adventurer": {
"type": "array",
"items": {
"type": "object",
"properties": {
"profileTarget": {
"nullable": false,
"type": "string",
"example": "XXX"
},
"region": {
"nullable": false,
"type": "string",
"enum": [
"EU",
"NA",
"SA"
]
}
}
}
},
"/adventurer/search": {
"type": "array",
"items": {
"type": "object",
"properties": {
"page": {
"deprecated": true,
"nullable": false,
"type": "number",
"example": 1
},
"query": {
"nullable": false,
"type": "string",
"example": "Apple"
},
"region": {
"nullable": false,
"type": "string",
"enum": [
"EU",
"NA",
"SA"
]
},
"searchType": {
"nullable": false,
"type": "string",
"enum": [
"familyName",
"characterName"
]
}
}
}
},
"/guild": {
"type": "array",
"items": {
"type": "object",
"properties": {
"guildName": {
"nullable": false,
"type": "string",
"example": "TumblrGirls"
},
"region": {
"nullable": false,
"type": "string",
"enum": [
"EU",
"NA",
"SA"
]
}
}
}
},
"/guild/search": {
"deprecated": true,
"type": "array",
"items": {
"type": "object",
"properties": {
"page": {
"nullable": false,
"type": "number",
"example": 1
},
"query": {
"nullable": false,
"type": "string",
"example": "TumblrGirls"
},
"region": {
"nullable": false,
"type": "string",
"enum": [
"EU",
"NA",
"SA"
]
}
}
}
}
}
}
}
}
}
}
}
}
}
}
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 19,10 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/sa-/slicefunk v0.1.4 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 67,8 @@ github.com/patrickmn/go-cache v2.1.0 incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/sa-/slicefunk v0.1.4 h1:fCgDllo0nYVywdREyJm53BQ5rfMW8pin57yNVpyPxNU=
github.com/sa-/slicefunk v0.1.4/go.mod h1:k0abNpV9EW8LIPl2 Hc9RiKsojKmsUhNNGFyMpjMTCI=
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
Expand All @@ -82,6 84,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5 Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
Expand Down
7 changes: 4 additions & 3 deletions handlers/GetAdventurerSearch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 18,8 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
page := validators.ValidatePageQueryParam(r.URL.Query()["page"])
query, queryOk := validators.ValidateAdventurerNameQueryParam(r.URL.Query()["query"])
region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"])
searchType := validators.ValidateSearchTypeQueryParam(r.URL.Query()["searchType"])
searchTypeQueryParam := r.URL.Query()["searchType"]
searchType := validators.ValidateSearchTypeQueryParam(searchTypeQueryParam)

if !queryOk || !regionOk {
giveBadRequestResponse(w)
Expand All @@ -33,7 34,7 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
query = strings.ToLower(query)

// Look for cached data, then run the scraper if needed
data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)})
data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, searchTypeQueryParam[0], fmt.Sprint(page)})
if !found {
data, status = scrapers.ScrapeAdventurerSearch(region, query, searchType, page)

Expand All @@ -46,7 47,7 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
return
}

date, expires = profileSearchCache.AddRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)}, data, status)
date, expires = profileSearchCache.AddRecord([]string{region, query, searchTypeQueryParam[0], fmt.Sprint(page)}, data, status)
}

w.Header().Set("Date", date)
Expand Down
57 changes: 57 additions & 0 deletions handlers/GetCache.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,57 @@
package handlers

import (
"encoding/json"
"net/http"
"strconv"
"strings"

sf "github.com/sa-/slicefunk"
)

func getParseCacheKey(cacheType string) func(string) map[string]interface{} {
return func(key string) map[string]interface{} {
parts := strings.Split(key, ",")

switch cacheType {
case "/adventurer":
return map[string]interface{}{
"region": parts[0],
"profileTarget": parts[1],
}
case "/adventurer/search":
page, _ := strconv.Atoi(parts[3])

return map[string]interface{}{
"region": parts[0],
"query": parts[1],
"searhType": parts[2],
"page": page,
}
case "/guild":
return map[string]interface{}{
"region": parts[0],
"guildName": parts[1],
}
case "/guild/search":
page, _ := strconv.Atoi(parts[2])

return map[string]interface{}{
"region": parts[0],
"query": parts[1],
"page": page,
}
default:
return nil
}
}
}

func GetCache(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
"/adventurer": sf.Map(profilesCache.GetKeys(), getParseCacheKey("/adventurer")),
"/adventurer/search": sf.Map(profileSearchCache.GetKeys(), getParseCacheKey("/adventurer/search")),
"/guild": sf.Map(guildProfilesCache.GetKeys(), getParseCacheKey("/guild")),
"/guild/search": sf.Map(guildSearchCache.GetKeys(), getParseCacheKey("/guild/search")),
})
}
2 changes: 1 addition & 1 deletion handlers/GetStatus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 10,7 @@ import (
)

var initTime = time.Now()
var version = "1.8.4"
var version = "1.9.0"

func GetStatus(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
Expand Down
1 change: 1 addition & 0 deletions httpServer/BuildServer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 16,7 @@ func BuildServer() *http.Server {
"/v1": handlers.GetStatus,
"/v1/adventurer": handlers.GetAdventurer,
"/v1/adventurer/search": handlers.GetAdventurerSearch,
"/v1/cache": handlers.GetCache,
"/v1/guild": handlers.GetGuild,
"/v1/guild/search": handlers.GetGuildSearch,
}, handlers.Catchall)
Expand Down

0 comments on commit 1571316

Please sign in to comment.