[feature] Default to WASM-based SQLite driver (#3053)

* [feature] Default to WASM-based SQLite driver

With 0.16 out this switches our default SQLite driver to the WASM-based
solution instead. So far the driver seems to perform just as well.
Switching our default should result in it getting a bit more testing
during the 0.17 development cycle.

* add the ol' john hancock

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>
This commit is contained in:
Daenney 2024-06-29 09:35:57 +02:00 committed by GitHub
parent 86786ae5b3
commit 137ef5a9ff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 111 additions and 111 deletions

View file

@ -42,7 +42,7 @@ steps:
go test go test
-failfast -failfast
-timeout=20m -timeout=20m
-tags "wasmsqlite3 netgo osusergo static_build kvformat timetzdata" -tags "netgo osusergo static_build kvformat timetzdata"
./... ./...
- ./test/envparsing.sh - ./test/envparsing.sh
- ./test/swagger.sh - ./test/swagger.sh
@ -204,6 +204,6 @@ steps:
--- ---
kind: signature kind: signature
hmac: 2e74313f4192b3e6daf6d1d00a7c3796019d93da7ce7e0a77208ccc3c37089b0 hmac: 86ebddcd630792cac43aa92fa7f45118943c51b5157491d05eb480ac21762329
... ...

View file

@ -30,7 +30,7 @@ builds:
- >- - >-
{{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }} {{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }}
- >- - >-
{{ if and (index .Env "WASMSQLITE3") (.Env.WASMSQLITE3) }}wasmsqlite3{{ end }} {{ if and (index .Env "MODERNCSQLITE3") (.Env.MODERNCSQLITE3) }}moderncsqlite3{{ end }}
env: env:
- CGO_ENABLED=0 - CGO_ENABLED=0
goos: goos:

View file

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !wasmsqlite3 //go:build !moderncsqlite3
package sqlite package sqlite
@ -23,19 +23,22 @@ import (
"context" "context"
"database/sql/driver" "database/sql/driver"
"modernc.org/sqlite"
"github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/ncruces/go-sqlite3"
sqlite3driver "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary
_ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs
) )
// Driver is our own wrapper around the // Driver is our own wrapper around the
// sqlite.Driver{} type in order to wrap // driver.SQLite{} type in order to wrap
// further SQL types with our own // further SQL types with our own
// functionality, e.g. err processing. // functionality, e.g. err processing.
type Driver struct{ sqlite.Driver } type Driver struct{ sqlite3driver.SQLite }
func (d *Driver) Open(name string) (driver.Conn, error) { func (d *Driver) Open(name string) (driver.Conn, error) {
conn, err := d.Driver.Open(name) conn, err := d.SQLite.Open(name)
if err != nil { if err != nil {
err = processSQLiteError(err) err = processSQLiteError(err)
return nil, err return nil, err
@ -43,6 +46,30 @@ func (d *Driver) Open(name string) (driver.Conn, error) {
return &sqliteConn{conn.(connIface)}, nil return &sqliteConn{conn.(connIface)}, nil
} }
func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
cc, err := d.SQLite.OpenConnector(name)
if err != nil {
return nil, err
}
return &sqliteConnector{driver: d, Connector: cc}, nil
}
type sqliteConnector struct {
driver *Driver
driver.Connector
}
func (c *sqliteConnector) Driver() driver.Driver { return c.driver }
func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) {
conn, err := c.Connector.Connect(ctx)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteConn{conn.(connIface)}, nil
}
type sqliteConn struct{ connIface } type sqliteConn struct{ connIface }
func (c *sqliteConn) Begin() (driver.Tx, error) { func (c *sqliteConn) Begin() (driver.Tx, error) {
@ -81,26 +108,16 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive
return return
} }
func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
return c.QueryContext(context.Background(), query, db.ToNamedValues(args))
}
func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) {
rows, err = c.connIface.QueryContext(ctx, query, args)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteRows{rows.(rowsIface)}, nil
}
func (c *sqliteConn) Close() (err error) { func (c *sqliteConn) Close() (err error) {
// Get acces the underlying raw sqlite3 conn.
raw := c.connIface.(sqlite3.DriverConn).Raw()
// see: https://www.sqlite.org/pragma.html#pragma_optimize // see: https://www.sqlite.org/pragma.html#pragma_optimize
const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;"
_, _ = c.connIface.ExecContext(context.Background(), onClose, nil) _ = raw.Exec(onClose)
// Finally, close the conn. // Finally, close.
err = c.connIface.Close() err = raw.Close()
return return
} }
@ -164,7 +181,7 @@ func (r *sqliteRows) Close() (err error) {
} }
// connIface is the driver.Conn interface // connIface is the driver.Conn interface
// types (and the like) that modernc.org/sqlite.conn // types (and the like) that go-sqlite3/driver.conn
// conforms to. Useful so you don't need // conforms to. Useful so you don't need
// to repeatedly perform checks yourself. // to repeatedly perform checks yourself.
type connIface interface { type connIface interface {
@ -172,11 +189,10 @@ type connIface interface {
driver.ConnBeginTx driver.ConnBeginTx
driver.ConnPrepareContext driver.ConnPrepareContext
driver.ExecerContext driver.ExecerContext
driver.QueryerContext
} }
// StmtIface is the driver.Stmt interface // StmtIface is the driver.Stmt interface
// types (and the like) that modernc.org/sqlite.stmt // types (and the like) that go-sqlite3/driver.stmt
// conforms to. Useful so you don't need // conforms to. Useful so you don't need
// to repeatedly perform checks yourself. // to repeatedly perform checks yourself.
type stmtIface interface { type stmtIface interface {
@ -186,12 +202,10 @@ type stmtIface interface {
} }
// RowsIface is the driver.Rows interface // RowsIface is the driver.Rows interface
// types (and the like) that modernc.org/sqlite.rows // types (and the like) that go-sqlite3/driver.rows
// conforms to. Useful so you don't need // conforms to. Useful so you don't need
// to repeatedly perform checks yourself. // to repeatedly perform checks yourself.
type rowsIface interface { type rowsIface interface {
driver.Rows driver.Rows
driver.RowsColumnTypeDatabaseTypeName driver.RowsColumnTypeDatabaseTypeName
driver.RowsColumnTypeLength
driver.RowsColumnTypeScanType
} }

View file

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build wasmsqlite3 //go:build moderncsqlite3
package sqlite package sqlite
@ -23,22 +23,19 @@ import (
"context" "context"
"database/sql/driver" "database/sql/driver"
"github.com/superseriousbusiness/gotosocial/internal/db" "modernc.org/sqlite"
"github.com/ncruces/go-sqlite3" "github.com/superseriousbusiness/gotosocial/internal/db"
sqlite3driver "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary
_ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs
) )
// Driver is our own wrapper around the // Driver is our own wrapper around the
// driver.SQLite{} type in order to wrap // sqlite.Driver{} type in order to wrap
// further SQL types with our own // further SQL types with our own
// functionality, e.g. err processing. // functionality, e.g. err processing.
type Driver struct{ sqlite3driver.SQLite } type Driver struct{ sqlite.Driver }
func (d *Driver) Open(name string) (driver.Conn, error) { func (d *Driver) Open(name string) (driver.Conn, error) {
conn, err := d.SQLite.Open(name) conn, err := d.Driver.Open(name)
if err != nil { if err != nil {
err = processSQLiteError(err) err = processSQLiteError(err)
return nil, err return nil, err
@ -46,30 +43,6 @@ func (d *Driver) Open(name string) (driver.Conn, error) {
return &sqliteConn{conn.(connIface)}, nil return &sqliteConn{conn.(connIface)}, nil
} }
func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
cc, err := d.SQLite.OpenConnector(name)
if err != nil {
return nil, err
}
return &sqliteConnector{driver: d, Connector: cc}, nil
}
type sqliteConnector struct {
driver *Driver
driver.Connector
}
func (c *sqliteConnector) Driver() driver.Driver { return c.driver }
func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) {
conn, err := c.Connector.Connect(ctx)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteConn{conn.(connIface)}, nil
}
type sqliteConn struct{ connIface } type sqliteConn struct{ connIface }
func (c *sqliteConn) Begin() (driver.Tx, error) { func (c *sqliteConn) Begin() (driver.Tx, error) {
@ -108,16 +81,26 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive
return return
} }
func (c *sqliteConn) Close() (err error) { func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
// Get acces the underlying raw sqlite3 conn. return c.QueryContext(context.Background(), query, db.ToNamedValues(args))
raw := c.connIface.(sqlite3.DriverConn).Raw() }
func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) {
rows, err = c.connIface.QueryContext(ctx, query, args)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteRows{rows.(rowsIface)}, nil
}
func (c *sqliteConn) Close() (err error) {
// see: https://www.sqlite.org/pragma.html#pragma_optimize // see: https://www.sqlite.org/pragma.html#pragma_optimize
const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;"
_ = raw.Exec(onClose) _, _ = c.connIface.ExecContext(context.Background(), onClose, nil)
// Finally, close. // Finally, close the conn.
err = raw.Close() err = c.connIface.Close()
return return
} }
@ -181,7 +164,7 @@ func (r *sqliteRows) Close() (err error) {
} }
// connIface is the driver.Conn interface // connIface is the driver.Conn interface
// types (and the like) that go-sqlite3/driver.conn // types (and the like) that modernc.org/sqlite.conn
// conforms to. Useful so you don't need // conforms to. Useful so you don't need
// to repeatedly perform checks yourself. // to repeatedly perform checks yourself.
type connIface interface { type connIface interface {
@ -189,10 +172,11 @@ type connIface interface {
driver.ConnBeginTx driver.ConnBeginTx
driver.ConnPrepareContext driver.ConnPrepareContext
driver.ExecerContext driver.ExecerContext
driver.QueryerContext
} }
// StmtIface is the driver.Stmt interface // StmtIface is the driver.Stmt interface
// types (and the like) that go-sqlite3/driver.stmt // types (and the like) that modernc.org/sqlite.stmt
// conforms to. Useful so you don't need // conforms to. Useful so you don't need
// to repeatedly perform checks yourself. // to repeatedly perform checks yourself.
type stmtIface interface { type stmtIface interface {
@ -202,10 +186,12 @@ type stmtIface interface {
} }
// RowsIface is the driver.Rows interface // RowsIface is the driver.Rows interface
// types (and the like) that go-sqlite3/driver.rows // types (and the like) that modernc.org/sqlite.rows
// conforms to. Useful so you don't need // conforms to. Useful so you don't need
// to repeatedly perform checks yourself. // to repeatedly perform checks yourself.
type rowsIface interface { type rowsIface interface {
driver.Rows driver.Rows
driver.RowsColumnTypeDatabaseTypeName driver.RowsColumnTypeDatabaseTypeName
driver.RowsColumnTypeLength
driver.RowsColumnTypeScanType
} }

View file

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !wasmsqlite3 //go:build !moderncsqlite3
package sqlite package sqlite
@ -23,9 +23,7 @@ import (
"database/sql/driver" "database/sql/driver"
"fmt" "fmt"
"modernc.org/sqlite" "github.com/ncruces/go-sqlite3"
sqlite3 "modernc.org/sqlite/lib"
"github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db"
) )
@ -33,30 +31,30 @@ import (
// handle conversion to any of our common db types. // handle conversion to any of our common db types.
func processSQLiteError(err error) error { func processSQLiteError(err error) error {
// Attempt to cast as sqlite error. // Attempt to cast as sqlite error.
sqliteErr, ok := err.(*sqlite.Error) sqliteErr, ok := err.(*sqlite3.Error)
if !ok { if !ok {
return err return err
} }
// Handle supplied error code: // Handle supplied error code:
switch sqliteErr.Code() { switch sqliteErr.ExtendedCode() {
case sqlite3.SQLITE_CONSTRAINT_UNIQUE, case sqlite3.CONSTRAINT_UNIQUE,
sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: sqlite3.CONSTRAINT_PRIMARYKEY:
return db.ErrAlreadyExists return db.ErrAlreadyExists
// Busy should be very rare, but // Busy should be very rare, but on
// on busy tell the database to close // busy tell the database to close the
// the connection, re-open and re-attempt // connection, re-open and re-attempt
// which should give a necessary timeout. // which should give necessary timeout.
case sqlite3.SQLITE_BUSY, case sqlite3.BUSY_RECOVERY,
sqlite3.SQLITE_BUSY_RECOVERY, sqlite3.BUSY_SNAPSHOT:
sqlite3.SQLITE_BUSY_SNAPSHOT:
return driver.ErrBadConn return driver.ErrBadConn
} }
// Wrap the returned error with the code and // Wrap the returned error with the code and
// extended code for easier debugging later. // extended code for easier debugging later.
return fmt.Errorf("%w (code=%d)", err, return fmt.Errorf("%w (code=%d extended=%d)", err,
sqliteErr.Code(), sqliteErr.Code(),
sqliteErr.ExtendedCode(),
) )
} }

View file

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build wasmsqlite3 //go:build moderncsqlite3
package sqlite package sqlite
@ -23,7 +23,9 @@ import (
"database/sql/driver" "database/sql/driver"
"fmt" "fmt"
"github.com/ncruces/go-sqlite3" "modernc.org/sqlite"
sqlite3 "modernc.org/sqlite/lib"
"github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db"
) )
@ -31,30 +33,30 @@ import (
// handle conversion to any of our common db types. // handle conversion to any of our common db types.
func processSQLiteError(err error) error { func processSQLiteError(err error) error {
// Attempt to cast as sqlite error. // Attempt to cast as sqlite error.
sqliteErr, ok := err.(*sqlite3.Error) sqliteErr, ok := err.(*sqlite.Error)
if !ok { if !ok {
return err return err
} }
// Handle supplied error code: // Handle supplied error code:
switch sqliteErr.ExtendedCode() { switch sqliteErr.Code() {
case sqlite3.CONSTRAINT_UNIQUE, case sqlite3.SQLITE_CONSTRAINT_UNIQUE,
sqlite3.CONSTRAINT_PRIMARYKEY: sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
return db.ErrAlreadyExists return db.ErrAlreadyExists
// Busy should be very rare, but on // Busy should be very rare, but
// busy tell the database to close the // on busy tell the database to close
// connection, re-open and re-attempt // the connection, re-open and re-attempt
// which should give necessary timeout. // which should give a necessary timeout.
case sqlite3.BUSY_RECOVERY, case sqlite3.SQLITE_BUSY,
sqlite3.BUSY_SNAPSHOT: sqlite3.SQLITE_BUSY_RECOVERY,
sqlite3.SQLITE_BUSY_SNAPSHOT:
return driver.ErrBadConn return driver.ErrBadConn
} }
// Wrap the returned error with the code and // Wrap the returned error with the code and
// extended code for easier debugging later. // extended code for easier debugging later.
return fmt.Errorf("%w (code=%d extended=%d)", err, return fmt.Errorf("%w (code=%d)", err,
sqliteErr.Code(), sqliteErr.Code(),
sqliteErr.ExtendedCode(),
) )
} }

View file

@ -22,7 +22,7 @@ GO_GCFLAGS=${GO_GCFLAGS-}
# - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability) # - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability)
# - debug: enables /debug/pprof endpoint (adds debug, at performance cost) # - debug: enables /debug/pprof endpoint (adds debug, at performance cost)
# - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost) # - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost)
# - wasmsqlite3: uses SQLite through WASM instead of the C-to-Go transpilation (experimental) # - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver (disables the WASM-based SQLite driver)
log_exec env CGO_ENABLED=0 go build -trimpath -v \ log_exec env CGO_ENABLED=0 go build -trimpath -v \
-tags "${GO_BUILDTAGS}" \ -tags "${GO_BUILDTAGS}" \
-ldflags="${GO_LDFLAGS}" \ -ldflags="${GO_LDFLAGS}" \