From f55b4468b3ccb09018fd0a16f1b75543b646c77f Mon Sep 17 00:00:00 2001 From: Siavash Sameni Date: Fri, 29 Aug 2025 08:17:52 +0400 Subject: [PATCH] initial release --- .gitignore | 52 +++++++ Dockerfile | 15 ++ go.mod | 27 ++++ go.sum | 124 +++++++++++++++++ internal/config/config.go | 43 ++++++ internal/fetcher/alchemy.go | 74 ++++++++++ internal/fetcher/rpc.go | 264 ++++++++++++++++++++++++++++++++++++ internal/store/store.go | 91 +++++++++++++ 8 files changed, 690 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/fetcher/alchemy.go create mode 100644 internal/fetcher/rpc.go create mode 100644 internal/store/store.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a674e68 --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +# Go build artifacts +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.test +*.out + +# Binaries and build dirs +bin/ +build/ +dist/ + +# Common local binaries (if built into tree) +cmd/**/nftcache + +# Coverage / test reports +coverage/ +cover* +*.cover +*.coverprofile +*.cov +*.lcov +coverage.*.out + +# Profiling / debug data +*.pprof +cpu.prof +mem.prof +heap.prof +profile* + +# Logs +logs/ +*.log + +# Environment files +.env +.env.local +.env.* + +# Editor/OS +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store +Thumbs.db + +# Vendored deps (if ever used) +vendor/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5cd6d18 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.22-alpine AS build +WORKDIR /app +COPY go.mod ./ +RUN go mod download +COPY . ./ +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /nftcache ./cmd/nftcache + +FROM gcr.io/distroless/base-debian12 +WORKDIR / +COPY --from=build /nftcache /nftcache +USER 65532:65532 +EXPOSE 8090 +ENV NFTCACHE_DATA_DIR=/data +ENTRYPOINT ["/nftcache"] diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..03d52e0 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module nftcache + +go 1.22.0 + +require ( + github.com/dgraph-io/badger/v4 v4.2.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/klauspost/compress v1.12.3 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + go.opencensus.io v0.22.5 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a9936e6 --- /dev/null +++ b/go.sum @@ -0,0 +1,124 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..9970bc6 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,43 @@ +package config + +import ( + "errors" + "io" + "os" + + yaml "gopkg.in/yaml.v3" +) + +type Contract struct { + Network string `yaml:"network"` + Address string `yaml:"address"` + MaxTokenId string `yaml:"max_token_id"` + FromBlock string `yaml:"from_block"` // kept for backward compatibility, maps to MaxTokenId +} + +type File struct { + Contracts map[string]Contract `yaml:"contracts"` +} + +func Load(path string) (*File, error) { + if path == "" { return &File{Contracts: map[string]Contract{}}, nil } + f, err := os.Open(path) + if err != nil { return nil, err } + defer f.Close() + b, err := io.ReadAll(f) + if err != nil { return nil, err } + var cfg File + if err := yaml.Unmarshal(b, &cfg); err != nil { return nil, err } + if cfg.Contracts == nil { cfg.Contracts = map[string]Contract{} } + return &cfg, nil +} + +func (f *File) Get(name string) (Contract, bool) { + c, ok := f.Contracts[name] + return c, ok +} + +func (c Contract) Validate() error { + if c.Network == "" || c.Address == "" { return errors.New("missing network or address") } + return nil +} diff --git a/internal/fetcher/alchemy.go b/internal/fetcher/alchemy.go new file mode 100644 index 0000000..4d593c3 --- /dev/null +++ b/internal/fetcher/alchemy.go @@ -0,0 +1,74 @@ +package fetcher + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" +) + +type AlchemyClient struct { + apiKey string + http *http.Client +} + +func NewAlchemyClient(apiKey string) *AlchemyClient { + return &AlchemyClient{apiKey: apiKey, http: &http.Client{}} +} + +// FetchOwnerTokens returns tokenIDs (as strings) for the owner for a single contract. +// network: e.g. "arb" (arbitrum one) -> uses arb-mainnet endpoint. +func (c *AlchemyClient) FetchOwnerTokens(network, contract, owner string) ([]string, error) { + if c.apiKey == "" { + return nil, errors.New("missing ALCHEMY_API_KEY") + } + base := networkBase(network) + if base == "" { + return nil, fmt.Errorf("unsupported network: %s", network) + } + endpoint := fmt.Sprintf("%s/nft/v3/%s/getNFTsForOwner", base, c.apiKey) + q := url.Values{} + q.Set("owner", owner) + q.Add("contractAddresses[]", contract) + q.Set("withMetadata", "false") + url := endpoint + "?" + q.Encode() + + resp, err := c.http.Get(url) + if err != nil { return nil, err } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("alchemy error: %s", resp.Status) + } + var parsed struct { + OwnedNfts []struct { + Id struct { TokenId string `json:"tokenId"` } `json:"id"` + } `json:"ownedNfts"` + } + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { return nil, err } + ids := make([]string, 0, len(parsed.OwnedNfts)) + for _, n := range parsed.OwnedNfts { + ids = append(ids, stripHexPrefix(n.Id.TokenId)) + } + return ids, nil +} + +func stripHexPrefix(s string) string { + s = strings.TrimPrefix(s, "0x") + // Alchemy may return hex with leading zeros; keep as-is (frontend can parse) + return s +} + +func networkBase(network string) string { + switch strings.ToLower(network) { + case "arb", "arbitrum", "arbitrum-one": + return "https://arb-mainnet.g.alchemy.com" + case "arb-sepolia", "arbitrum-sepolia": + return "https://arb-sepolia.g.alchemy.com" + case "eth", "mainnet", "ethereum": + return "https://eth-mainnet.g.alchemy.com" + default: + return "" + } +} diff --git a/internal/fetcher/rpc.go b/internal/fetcher/rpc.go new file mode 100644 index 0000000..d3db740 --- /dev/null +++ b/internal/fetcher/rpc.go @@ -0,0 +1,264 @@ +package fetcher + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" +) + +type RPCClient struct { + // Map of network key -> RPC URL, e.g. {"eth": "https://mainnet.infura.io/v3/...", "arb": "https://arb1...", "base": "https://base-mainnet..."} + RPC map[string]string + http *http.Client + rateLimiter chan struct{} // Rate limiter: 5 TPS +} + +func NewRPCClient(rpc map[string]string) *RPCClient { + client := &RPCClient{ + RPC: rpc, + http: &http.Client{}, + rateLimiter: make(chan struct{}, 5), // 5 TPS rate limiter + } + + // Fill the rate limiter initially + for i := 0; i < 5; i++ { + client.rateLimiter <- struct{}{} + } + + // Refill rate limiter at 5 TPS (200ms intervals) + go func() { + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + select { + case client.rateLimiter <- struct{}{}: + default: + // Channel is full, skip + } + } + }() + + return client +} + +// FetchOwnerTokens iterates through tokenIds 0,1,2,... up to maxTokenId +// and calls ownerOf for each to find which ones belong to owner. +// network: key like "eth", "arb", "base". +// contract: 0x... +// owner: 0x... +// maxTokenId: maximum tokenId to check +// FetchAllTokenOwners scans all tokens 0..maxTokenId and returns ownership map +func (c *RPCClient) FetchAllTokenOwners(network, contract string, maxTokenId int, debug bool) (map[string]string, error) { + rpcURL := c.RPC[strings.ToLower(network)] + if rpcURL == "" { return nil, fmt.Errorf("missing RPC for network %s", network) } + + contract = strings.ToLower(contract) + owners := make(map[string]string) + consecutiveErrors := 0 + maxConsecutiveErrors := 50 // Stop if 50 consecutive tokens don't exist + + // Check each tokenId from 0 to maxTokenId + for i := 0; i < maxTokenId; i++ { + tokenOwner, err := c.getOwnerOf(rpcURL, contract, i, debug) + if err != nil { + // If we hit the first invalid token ID, stop immediately as requested + if strings.Contains(strings.ToLower(err.Error()), "invalid token id") { + if debug { + fmt.Printf("DEBUG: Stopping at first invalid token ID %d due to error: %v\n", i, err) + } + break + } + consecutiveErrors++ + if debug { + fmt.Printf("DEBUG: Token %d error: %v (consecutive errors: %d)\n", i, err, consecutiveErrors) + } + // Early termination if too many consecutive errors + if consecutiveErrors >= maxConsecutiveErrors { + if debug { + fmt.Printf("DEBUG: Stopping scan at token %d due to %d consecutive errors\n", i, consecutiveErrors) + } + break + } + continue + } + + consecutiveErrors = 0 // Reset on successful call + // Store tokenId -> canonicalized owner + canonicalOwner := canonicalizeAddr(tokenOwner) + owners[strconv.Itoa(i)] = canonicalOwner + + // Log specific tokens for debugging + if debug && (i == 103 || i == 0 || i == 1 || i%100 == 0 || len(owners) <= 50) { + fmt.Printf("DEBUG: Token %d owner: %s (canonical: %s)\n", i, tokenOwner, canonicalOwner) + } + + // Always log token 103 regardless of debug mode + if i == 103 { + fmt.Printf("ALWAYS: Token 103 owner: %s (canonical: %s)\n", tokenOwner, canonicalOwner) + } + } + + fmt.Printf("DEBUG: Scanned up to token %d, found %d with owners\n", maxTokenId, len(owners)) + return owners, nil +} + +func (c *RPCClient) FetchOwnerTokens(network, contract, owner string, maxTokenId int) ([]string, error) { + rpcURL := c.RPC[strings.ToLower(network)] + if rpcURL == "" { return nil, fmt.Errorf("missing RPC for network %s", network) } + + contract = strings.ToLower(contract) + owner = strings.ToLower(owner) + + var owned []string + + // Check each tokenId from 0 to maxTokenId + for i := 0; i < maxTokenId; i++ { + tokenOwner, err := c.getOwnerOf(rpcURL, contract, i, false) // No debug for this method + if err != nil { + // Token might not exist, skip + continue + } + if strings.EqualFold(tokenOwner, owner) { + owned = append(owned, strconv.Itoa(i)) + } + } + + return owned, nil +} + +// canonicalizeAddr normalizes address to 0x + 40 lowercase hex +func canonicalizeAddr(addr string) string { + x := strings.ToLower(strings.TrimSpace(addr)) + if strings.HasPrefix(x, "0x") { x = x[2:] } + if len(x) > 40 { x = x[len(x)-40:] } + if len(x) < 40 { x = strings.Repeat("0", 40-len(x)) + x } + return "0x" + x +} + +// getOwnerOf calls ownerOf(tokenId) on the contract with rate limiting and retry logic +func (c *RPCClient) getOwnerOf(rpcURL, contract string, tokenId int, debug bool) (string, error) { + maxRetries := 10 + baseDelay := 100 * time.Millisecond + + for attempt := 0; attempt <= maxRetries; attempt++ { + // Wait for rate limiter + <-c.rateLimiter + + if debug && attempt > 0 { + fmt.Printf("DEBUG: Retry attempt %d for token %d\n", attempt, tokenId) + } + + result, err := c.makeRPCCall(rpcURL, contract, tokenId, debug) + if err != nil { + // Check if it's a 429 error + if strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "Too Many Requests") { + if attempt < maxRetries { + // Exponential backoff: 100ms, 200ms, 400ms, 800ms, etc. + delay := time.Duration(1<= 42 { + owner := "0x" + result.Result[len(result.Result)-40:] + if debug { + fmt.Printf("DEBUG: Extracted owner tokenId=%d owner=%s\n", tokenId, owner) + } + return owner, nil + } + + if debug { + fmt.Printf("DEBUG: Invalid response length tokenId=%d result=%s\n", tokenId, result.Result) + } + return "", fmt.Errorf("invalid ownerOf response") +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..47f7295 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,91 @@ +package store + +import ( + "encoding/json" + "errors" + "time" + + badger "github.com/dgraph-io/badger/v4" +) + +type Entry struct { + TokenIDs []string `json:"token_ids"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ContractEntry stores all token ownership data for a contract +type ContractEntry struct { + // Map of tokenId -> owner address (canonical) + TokenOwners map[string]string `json:"token_owners"` + UpdatedAt time.Time `json:"updated_at"` + MaxScanned int `json:"max_scanned"` +} + +type Cache struct { + db *badger.DB +} + +func New(dir string) (*Cache, error) { + opts := badger.DefaultOptions(dir) + opts.Logger = nil + db, err := badger.Open(opts) + if err != nil { return nil, err } + return &Cache{db: db}, nil +} + +func (c *Cache) Close() error { return c.db.Close() } + +func (c *Cache) Get(key string) (*Entry, bool) { + var e Entry + err := c.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { return err } + return item.Value(func(v []byte) error { return json.Unmarshal(v, &e) }) + }) + if err != nil { return nil, false } + return &e, true +} + +func (c *Cache) Set(key string, ids []string) error { + val, _ := json.Marshal(Entry{TokenIDs: ids, UpdatedAt: time.Now().UTC()}) + return c.db.Update(func(txn *badger.Txn) error { + return txn.Set([]byte(key), val) + }) +} + +// SetContract stores all token ownership data for a contract +func (c *Cache) SetContract(key string, tokenOwners map[string]string, maxScanned int) error { + val, _ := json.Marshal(ContractEntry{ + TokenOwners: tokenOwners, + UpdatedAt: time.Now().UTC(), + MaxScanned: maxScanned, + }) + return c.db.Update(func(txn *badger.Txn) error { + return txn.Set([]byte(key), val) + }) +} + +// GetContract retrieves contract ownership data +func (c *Cache) GetContract(key string) (*ContractEntry, bool) { + var e ContractEntry + err := c.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { return err } + return item.Value(func(v []byte) error { return json.Unmarshal(v, &e) }) + }) + if err != nil { return nil, false } + return &e, true +} + +func (c *Cache) Delete(key string) error { + return c.db.Update(func(txn *badger.Txn) error { + err := txn.Delete([]byte(key)) + if errors.Is(err, badger.ErrKeyNotFound) { return nil } + return err + }) +} + +// DeleteContract removes contract cache entry +func (c *Cache) DeleteContract(key string) error { + return c.Delete(key) +}