Skip to content

Creating pmtiles outputter #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Creating pmtiles outputter #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7fc6a4d
Creating pmtiles outputter
iandees Mar 31, 2025
ae6d573
Specify correct tile type and seek to the actual beginning of the til…
iandees Mar 31, 2025
59be445
Add ensuregzip warning for pmtiles path too
iandees Mar 31, 2025
49f1035
Better handling of compression, sort entries, don't flip Y
iandees Mar 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
37 changes: 31 additions & 6 deletions cmd/build/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,15 @@ func processResults(results chan *tilepack.TileResponse, processor tilepack.Tile
func main() {
generatorStr := flag.String("generator", "xyz", "Which tile fetcher to use. Options are xyz, metatile, tapalcatl2.")
fileTransportRoot := flag.String("file-transport-root", "", "The root directory for tiles if -url-template defines a file:// URL scheme")
outputMode := flag.String("output-mode", "mbtiles", "Valid modes are: disk, mbtiles.")
outputMode := flag.String("output-mode", "mbtiles", "Valid modes are: disk, mbtiles, pmtiles.")
outputDSN := flag.String("dsn", "", "Path, or DSN string, to output files.")
boundingBoxStr := flag.String("bounds", "-90.0,-180.0,90.0,180.0", "Comma-separated bounding box in south,west,north,east format. Defaults to the whole world.")
zoomsStr := flag.String("zooms", "0,1,2,3,4,5,6,7,8,9,10", "Comma-separated list of zoom levels or a '{MIN_ZOOM}-{MAX_ZOOM}' range string.")
numTileFetchWorkers := flag.Int("workers", 25, "Number of tile fetch workers to use.")
mbtilesBatchSize := flag.Int("batch-size", 50, "(For mbtiles outputter) Number of tiles to batch together before writing to mbtiles")
mbtilesTilesetName := flag.String("tileset-name", "tileset", "(For mbtiles outputter) Name of the tileset to write to the mbtiles file metadata.")
mbtilesFormat := flag.String("mbtiles-format", "pbf", "(For mbtiles outputter) Format of the tiles in the mbtiles file metadata.")
mbtilesFormat := flag.String("mbtiles-format", "pbf", "(Deprecated. Use --output-format instead)")
outputFormat := flag.String("output-format", "pbf", "(For mbtiles and pmtiles outputter) Format of the tiles in the mbtiles or pmtiles file metadata.")
requestTimeout := flag.Int("timeout", 60, "HTTP client timeout for tile requests.")
cpuProfile := flag.String("cpuprofile", "", "Enables CPU profiling. Saves the dump to the given path.")
invertedY := flag.Bool("inverted-y", false, "Invert the Y-value of tiles to match the TMS (as opposed to ZXY) tile format.")
Expand Down Expand Up @@ -235,12 +236,18 @@ func main() {
case "mbtiles":
metadata := tilepack.NewMbtilesMetadata(map[string]string{})

if *mbtilesFormat == "" {
log.Fatalf("--mbtiles-format is required for mbtiles output")
// mbtilesFormat is deprecated, use outputFormat instead
if *mbtilesFormat != "" {
log.Printf("Warning: --mbtiles-format is deprecated, use --output-format instead")
*outputFormat = *mbtilesFormat
}
metadata.Set("format", *mbtilesFormat)

if *mbtilesFormat != "pbf" && *ensureGzip {
if *outputFormat == "" {
log.Fatalf("--output-format is required for mbtiles output")
}
metadata.Set("format", *outputFormat)

if *outputFormat != "pbf" && *ensureGzip {
log.Printf("Warning: gzipping is only required for PBF tiles. You may want to disable it for other formats with --ensure-gzip=false")
}

Expand All @@ -250,6 +257,24 @@ func main() {
metadata.Set("name", *mbtilesTilesetName)

outputter, outputterErr = tilepack.NewMbtilesOutputter(*outputDSN, *mbtilesBatchSize, metadata)
case "pmtiles":
metadata := tilepack.NewMbtilesMetadata(map[string]string{})

if *outputFormat == "" {
log.Fatalf("--output-format is required for pmtiles output")
}
metadata.Set("format", *outputFormat)

if *outputFormat != "pbf" && *ensureGzip {
log.Printf("Warning: gzipping is only required for PBF tiles. You may want to disable it for other formats with --ensure-gzip=false")
}

if *mbtilesTilesetName == "" {
log.Fatalf("--tileset-name is required for pmtiles output")
}
metadata.Set("name", *mbtilesTilesetName)

outputter, outputterErr = tilepack.NewPmtilesOutputter(*outputDSN, *outputFormat, metadata)
default:
log.Fatalf("Unknown outputter: %s", *outputMode)
}
Expand Down
72 changes: 70 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,82 @@ require (
github.com/aws/aws-sdk-go v1.55.5
github.com/mattn/go-sqlite3 v1.14.23
github.com/paulmach/orb v0.11.1
github.com/protomaps/go-pmtiles v1.27.0
github.com/schollz/progressbar/v3 v3.14.6
)

require (
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.8.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/iam v1.1.13 // indirect
cloud.google.com/go/storage v1.43.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
github.com/RoaringBitmap/roaring v1.5.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rs/cors v1.11.1 // indirect
go.mongodb.org/mongo-driver v1.11.4 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
gocloud.dev v0.40.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/oauth2 v0.22.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
google.golang.org/api v0.191.0 // indirect
google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
modernc.org/libc v1.41.0 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.7.2 // indirect
modernc.org/sqlite v1.29.1 // indirect
zombiezen.com/go/sqlite v1.1.2 // indirect
)
Loading