Commit ef4c52da authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson

Update kine to v0.5.0

parent ed9f305a
......@@ -90,9 +90,9 @@ require (
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1
github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
github.com/lib/pq v1.1.1
github.com/lib/pq v1.8.0
github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455
github.com/mattn/go-sqlite3 v1.13.0
github.com/mattn/go-sqlite3 v1.14.4
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/opencontainers/runc v1.0.0-rc9
github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52
......@@ -100,7 +100,7 @@ require (
github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 // indirect
github.com/rancher/dynamiclistener v0.2.0
github.com/rancher/helm-controller v0.4.1
github.com/rancher/kine v0.4.0
github.com/rancher/kine v0.5.0
github.com/rancher/remotedialer v0.2.0
github.com/rancher/wrangler v0.4.0
github.com/rancher/wrangler-api v0.4.0
......
......@@ -544,8 +544,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kubernetes-client/go v0.0.0-20190516163813-075b33afc74f/go.mod h1:ks4KCmmxdXksTSu2dlnUanEOqNd/dsoyS6/7bay2RQ8=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
......@@ -582,9 +582,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-shellwords v1.0.5 h1:JhhFTIOslh5ZsPrpa3Wdg8bF0WI3b44EMblmU9wIsXc=
github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.13.0 h1:LnJI81JidiW9r7pS/hXe6cFeO5EXNq7KbfvoJLRI69c=
github.com/mattn/go-sqlite3 v1.13.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI=
github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
......@@ -717,8 +716,8 @@ github.com/rancher/helm-controller v0.4.1 h1:AbI3zXy2Qtw4e4A6IkxKozgnm6T3KKnUo6E
github.com/rancher/helm-controller v0.4.1/go.mod h1:194LHuZRrxcD82bG1rJtOWsw98U4JbPhDWqvL7l3PAw=
github.com/rancher/juju-to-pkg-errors v0.0.0-20200701001603-16f3c28b59bd h1:KPnQuOFWU6Jm7dTadhZ9TCVf6Y5NxoSyUjwkbD7En5Q=
github.com/rancher/juju-to-pkg-errors v0.0.0-20200701001603-16f3c28b59bd/go.mod h1:QYmg8cqWPPfIbpEuhtJbEdWwA6PEKSY016Z6EdfL9+8=
github.com/rancher/kine v0.4.0 h1:1IhWy3TzjExG8xnj46eyUEWdzqNAD1WrgL4eEBKm6Uc=
github.com/rancher/kine v0.4.0/go.mod h1:IImtCJ68AIkE+VY/kUI0NkyJL5q5WzO8QvMsSXqbrpA=
github.com/rancher/kine v0.5.0 h1:ot9ZInMCb0482aWfvO+3gI2B+e9vGxpY12EDJgpowiY=
github.com/rancher/kine v0.5.0/go.mod h1:NoqDMfN0Q+Wu23Kk3MfXfgLO2fE6abLaetejZs9HAYo=
github.com/rancher/kubernetes v1.17.13-k3s1 h1:omEWOhFQhnDCiVITq/Jj7cOMyGVUPynjWka9TqV3oX8=
github.com/rancher/kubernetes v1.17.13-k3s1/go.mod h1:J8bqjhrOYZ/dDhaQU8sEfM+aH81x3Urfw1uODZc7ZfU=
github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.13-k3s1 h1:rUGnhWZuOiRkCbsfTBSxAFxd/8UZtnEv8Xltr7sIHsQ=
......
......@@ -70,17 +70,4 @@ postgresql_uninstall() {
sudo rm -rf /var/lib/postgresql
}
megacheck_install() {
# Lock megacheck version at $MEGACHECK_VERSION to prevent spontaneous
# new error messages in old code.
go get -d honnef.co/go/tools/...
git -C $GOPATH/src/honnef.co/go/tools/ checkout $MEGACHECK_VERSION
go install honnef.co/go/tools/cmd/megacheck
megacheck --version
}
golint_install() {
go get golang.org/x/lint/golint
}
$1
language: go
go:
- 1.9.x
- 1.10.x
- 1.11.x
- 1.13.x
- 1.14.x
- master
sudo: true
......@@ -14,16 +13,11 @@ env:
- PQGOSSLTESTS=1
- PQSSLCERTTEST_PATH=$PWD/certs
- PGHOST=127.0.0.1
- MEGACHECK_VERSION=2017.2.2
matrix:
- PGVERSION=10
- PGVERSION=9.6
- PGVERSION=9.5
- PGVERSION=9.4
- PGVERSION=9.3
- PGVERSION=9.2
- PGVERSION=9.1
- PGVERSION=9.0
before_install:
- ./.travis.sh postgresql_uninstall
......@@ -31,9 +25,9 @@ before_install:
- ./.travis.sh postgresql_install
- ./.travis.sh postgresql_configure
- ./.travis.sh client_configure
- ./.travis.sh megacheck_install
- ./.travis.sh golint_install
- go get golang.org/x/tools/cmd/goimports
- go get golang.org/x/lint/golint
- GO111MODULE=on go get honnef.co/go/tools/cmd/staticcheck@2020.1.3
before_script:
- createdb pqgotest
......@@ -44,7 +38,7 @@ script:
- >
goimports -d -e $(find -name '*.go') | awk '{ print } END { exit NR == 0 ? 0 : 1 }'
- go vet ./...
- megacheck -go 1.9 ./...
- staticcheck -go 1.13 ./...
- golint ./...
- PQTEST_BINARY_PARAMETERS=no go test -race -v ./...
- PQTEST_BINARY_PARAMETERS=yes go test -race -v ./...
## Contributing to pq
`pq` has a backlog of pull requests, but contributions are still very
much welcome. You can help with patch review, submitting bug reports,
or adding new functionality. There is no formal style guide, but
please conform to the style of existing code and general Go formatting
conventions when submitting patches.
### Patch review
Help review existing open pull requests by commenting on the code or
proposed functionality.
### Bug reports
We appreciate any bug reports, but especially ones with self-contained
(doesn't depend on code outside of pq), minimal (can't be simplified
further) test cases. It's especially helpful if you can submit a pull
request with just the failing test case (you'll probably want to
pattern it after the tests in
[conn_test.go](https://github.com/lib/pq/blob/master/conn_test.go).
### New functionality
There are a number of pending patches for new functionality, so
additional feature patches will take a while to merge. Still, patches
are generally reviewed based on usefulness and complexity in addition
to time-in-queue, so if you have a knockout idea, take a shot. Feel
free to open an issue discussion your proposed patch beforehand.
# pq - A pure Go postgres driver for Go's database/sql package
[![GoDoc](https://godoc.org/github.com/lib/pq?status.svg)](https://godoc.org/github.com/lib/pq)
[![Build Status](https://travis-ci.org/lib/pq.svg?branch=master)](https://travis-ci.org/lib/pq)
[![GoDoc](https://godoc.org/github.com/lib/pq?status.svg)](https://pkg.go.dev/github.com/lib/pq?tab=doc)
## Install
go get github.com/lib/pq
## Docs
For detailed documentation and basic usage examples, please see the package
documentation at <https://godoc.org/github.com/lib/pq>.
## Tests
`go test` is used for testing. See [TESTS.md](TESTS.md) for more details.
## Features
* SSL
......@@ -29,67 +19,12 @@ documentation at <https://godoc.org/github.com/lib/pq>.
* Unix socket support
* Notifications: `LISTEN`/`NOTIFY`
* pgpass support
* GSS (Kerberos) auth
## Future / Things you can help with
* Better COPY FROM / COPY TO (see discussion in #181)
## Tests
## Thank you (alphabetical)
`go test` is used for testing. See [TESTS.md](TESTS.md) for more details.
Some of these contributors are from the original library `bmizerany/pq.go` whose
code still exists in here.
## Status
* Andy Balholm (andybalholm)
* Ben Berkert (benburkert)
* Benjamin Heatwole (bheatwole)
* Bill Mill (llimllib)
* Bjørn Madsen (aeons)
* Blake Gentry (bgentry)
* Brad Fitzpatrick (bradfitz)
* Charlie Melbye (cmelbye)
* Chris Bandy (cbandy)
* Chris Gilling (cgilling)
* Chris Walsh (cwds)
* Dan Sosedoff (sosedoff)
* Daniel Farina (fdr)
* Eric Chlebek (echlebek)
* Eric Garrido (minusnine)
* Eric Urban (hydrogen18)
* Everyone at The Go Team
* Evan Shaw (edsrzf)
* Ewan Chou (coocood)
* Fazal Majid (fazalmajid)
* Federico Romero (federomero)
* Fumin (fumin)
* Gary Burd (garyburd)
* Heroku (heroku)
* James Pozdena (jpoz)
* Jason McVetta (jmcvetta)
* Jeremy Jay (pbnjay)
* Joakim Sernbrant (serbaut)
* John Gallagher (jgallagher)
* Jonathan Rudenberg (titanous)
* Joël Stemmer (jstemmer)
* Kamil Kisiel (kisielk)
* Kelly Dunn (kellydunn)
* Keith Rarick (kr)
* Kir Shatrov (kirs)
* Lann Martin (lann)
* Maciek Sakrejda (uhoh-itsmaciek)
* Marc Brinkmann (mbr)
* Marko Tiikkaja (johto)
* Matt Newberry (MattNewberry)
* Matt Robenolt (mattrobenolt)
* Martin Olsen (martinolsen)
* Mike Lewis (mikelikespie)
* Nicolas Patry (Narsil)
* Oliver Tonnhofer (olt)
* Patrick Hayes (phayes)
* Paul Hammond (paulhammond)
* Ryan Smith (ryandotsmith)
* Samuel Stauffer (samuel)
* Timothée Peignier (cyberdelia)
* Travis Cline (tmc)
* TruongSinh Tran-Nguyen (truongsinh)
* Yaismel Miranda (ympons)
* notedit (notedit)
This package is effectively in maintenance mode and is not actively developed. Small patches and features are only rarely reviewed and merged. We recommend using [pgx](https://github.com/jackc/pgx) which is actively maintained.
......@@ -66,7 +66,7 @@ func (b *writeBuf) int16(n int) {
}
func (b *writeBuf) string(s string) {
b.buf = append(b.buf, (s + "\000")...)
b.buf = append(append(b.buf, s...), '\000')
}
func (b *writeBuf) byte(c byte) {
......
......@@ -92,6 +92,7 @@ type Dialer interface {
DialTimeout(network, address string, timeout time.Duration) (net.Conn, error)
}
// DialerContext is the context-aware dialer interface.
type DialerContext interface {
DialContext(ctx context.Context, network, address string) (net.Conn, error)
}
......@@ -148,6 +149,15 @@ type conn struct {
// If true this connection is in the middle of a COPY
inCopy bool
// If not nil, notices will be synchronously sent here
noticeHandler func(*Error)
// If not nil, notifications will be synchronously sent here
notificationHandler func(*Notification)
// GSSAPI context
gss GSS
}
// Handle driver-side settings in parsed connection string.
......@@ -301,6 +311,9 @@ func (c *Connector) open(ctx context.Context) (cn *conn, err error) {
err = cn.ssl(o)
if err != nil {
if cn.c != nil {
cn.c.Close()
}
return nil, err
}
......@@ -325,10 +338,6 @@ func (c *Connector) open(ctx context.Context) (cn *conn, err error) {
func dial(ctx context.Context, d Dialer, o values) (net.Conn, error) {
network, address := network(o)
// SSL is not necessary or supported over UNIX domain sockets
if network == "unix" {
o["sslmode"] = "disable"
}
// Zero or not specified means wait indefinitely.
if timeout, ok := o["connect_timeout"]; ok && timeout != "0" {
......@@ -546,7 +555,7 @@ func (cn *conn) Commit() (err error) {
// would get the same behaviour if you issued a COMMIT in a failed
// transaction, so it's also the least surprising thing to do here.
if cn.txnStatus == txnStatusInFailedTransaction {
if err := cn.Rollback(); err != nil {
if err := cn.rollback(); err != nil {
return err
}
return ErrInFailedTransaction
......@@ -573,7 +582,10 @@ func (cn *conn) Rollback() (err error) {
return driver.ErrBadConn
}
defer cn.errRecover(&err)
return cn.rollback()
}
func (cn *conn) rollback() (err error) {
cn.checkIsInTransaction(true)
_, commandTag, err := cn.simpleExec("ROLLBACK")
if err != nil {
......@@ -964,7 +976,13 @@ func (cn *conn) recv() (t byte, r *readBuf) {
case 'E':
panic(parseError(r))
case 'N':
// ignore
if n := cn.noticeHandler; n != nil {
n(parseError(r))
}
case 'A':
if n := cn.notificationHandler; n != nil {
n(recvNotification(r))
}
default:
return
}
......@@ -981,8 +999,14 @@ func (cn *conn) recv1Buf(r *readBuf) byte {
}
switch t {
case 'A', 'N':
// ignore
case 'A':
if n := cn.notificationHandler; n != nil {
n(recvNotification(r))
}
case 'N':
if n := cn.noticeHandler; n != nil {
n(parseError(r))
}
case 'S':
cn.processParameterStatus(r)
default:
......@@ -1050,7 +1074,10 @@ func isDriverSetting(key string) bool {
return true
case "binary_parameters":
return true
case "krbsrvname":
return true
case "krbspn":
return true
default:
return false
}
......@@ -1130,6 +1157,59 @@ func (cn *conn) auth(r *readBuf, o values) {
if r.int32() != 0 {
errorf("unexpected authentication response: %q", t)
}
case 7: // GSSAPI, startup
if newGss == nil {
errorf("kerberos error: no GSSAPI provider registered (import github.com/lib/pq/auth/kerberos if you need Kerberos support)")
}
cli, err := newGss()
if err != nil {
errorf("kerberos error: %s", err.Error())
}
var token []byte
if spn, ok := o["krbspn"]; ok {
// Use the supplied SPN if provided..
token, err = cli.GetInitTokenFromSpn(spn)
} else {
// Allow the kerberos service name to be overridden
service := "postgres"
if val, ok := o["krbsrvname"]; ok {
service = val
}
token, err = cli.GetInitToken(o["host"], service)
}
if err != nil {
errorf("failed to get Kerberos ticket: %q", err)
}
w := cn.writeBuf('p')
w.bytes(token)
cn.send(w)
// Store for GSSAPI continue message
cn.gss = cli
case 8: // GSSAPI continue
if cn.gss == nil {
errorf("GSSAPI protocol error")
}
b := []byte(*r)
done, tokOut, err := cn.gss.Continue(b)
if err == nil && !done {
w := cn.writeBuf('p')
w.bytes(tokOut)
cn.send(w)
}
// Errors fall through and read the more detailed message
// from the server..
case 10:
sc := scram.NewClient(sha256.New, o["user"], o["password"])
sc.Step(nil)
......@@ -1500,6 +1580,39 @@ func QuoteIdentifier(name string) string {
return `"` + strings.Replace(name, `"`, `""`, -1) + `"`
}
// QuoteLiteral quotes a 'literal' (e.g. a parameter, often used to pass literal
// to DDL and other statements that do not accept parameters) to be used as part
// of an SQL statement. For example:
//
// exp_date := pq.QuoteLiteral("2023-01-05 15:00:00Z")
// err := db.Exec(fmt.Sprintf("CREATE ROLE my_user VALID UNTIL %s", exp_date))
//
// Any single quotes in name will be escaped. Any backslashes (i.e. "\") will be
// replaced by two backslashes (i.e. "\\") and the C-style escape identifier
// that PostgreSQL provides ('E') will be prepended to the string.
func QuoteLiteral(literal string) string {
// This follows the PostgreSQL internal algorithm for handling quoted literals
// from libpq, which can be found in the "PQEscapeStringInternal" function,
// which is found in the libpq/fe-exec.c source file:
// https://git.postgresql.org/gitweb/?p=postgresql.git;a=blob;f=src/interfaces/libpq/fe-exec.c
//
// substitute any single-quotes (') with two single-quotes ('')
literal = strings.Replace(literal, `'`, `''`, -1)
// determine if the string has any backslashes (\) in it.
// if it does, replace any backslashes (\) with two backslashes (\\)
// then, we need to wrap the entire string with a PostgreSQL
// C-style escape. Per how "PQEscapeStringInternal" handles this case, we
// also add a space before the "E"
if strings.Contains(literal, `\`) {
literal = strings.Replace(literal, `\`, `\\`, -1)
literal = ` E'` + literal + `'`
} else {
// otherwise, we can just wrap the literal with a pair of single quotes
literal = `'` + literal + `'`
}
return literal
}
func md5s(s string) string {
h := md5.New()
h.Write([]byte(s))
......
......@@ -79,7 +79,7 @@ func (cn *conn) Ping(ctx context.Context) error {
if finish := cn.watchCancel(ctx); finish != nil {
defer finish()
}
rows, err := cn.simpleQuery("SELECT 'lib/pq ping test';")
rows, err := cn.simpleQuery(";")
if err != nil {
return driver.ErrBadConn // https://golang.org/pkg/database/sql/driver/#Pinger
}
......
......@@ -27,7 +27,7 @@ func (c *Connector) Connect(ctx context.Context) (driver.Conn, error) {
return c.open(ctx)
}
// Driver returnst the underlying driver of this Connector.
// Driver returns the underlying driver of this Connector.
func (c *Connector) Driver() driver.Driver {
return &Driver{}
}
......@@ -106,5 +106,10 @@ func NewConnector(dsn string) (*Connector, error) {
o["user"] = u
}
// SSL is not necessary or supported over UNIX domain sockets
if network, _ := network(o); network == "unix" {
o["sslmode"] = "disable"
}
return &Connector{opts: o, dialer: defaultDialer{}}, nil
}
......@@ -49,6 +49,7 @@ type copyin struct {
buffer []byte
rowData chan []byte
done chan bool
driver.Result
closed bool
......@@ -151,8 +152,12 @@ func (ci *copyin) resploop() {
switch t {
case 'C':
// complete
res, _ := ci.cn.parseComplete(r.string())
ci.setResult(res)
case 'N':
// NoticeResponse
if n := ci.cn.noticeHandler; n != nil {
n(parseError(&r))
}
case 'Z':
ci.cn.processReadyForQuery(&r)
ci.done <- true
......@@ -199,6 +204,22 @@ func (ci *copyin) setError(err error) {
ci.Unlock()
}
func (ci *copyin) setResult(result driver.Result) {
ci.Lock()
ci.Result = result
ci.Unlock()
}
func (ci *copyin) getResult() driver.Result {
ci.Lock()
result := ci.Result
if result == nil {
return driver.RowsAffected(0)
}
ci.Unlock()
return result
}
func (ci *copyin) NumInput() int {
return -1
}
......@@ -229,7 +250,11 @@ func (ci *copyin) Exec(v []driver.Value) (r driver.Result, err error) {
}
if len(v) == 0 {
return nil, ci.Close()
if err := ci.Close(); err != nil {
return driver.RowsAffected(0), err
}
return ci.getResult(), nil
}
numValues := len(v)
......
......@@ -241,5 +241,28 @@ bytes by the PostgreSQL server.
You can find a complete, working example of Listener usage at
https://godoc.org/github.com/lib/pq/example/listen.
Kerberos Support
If you need support for Kerberos authentication, add the following to your main
package:
import "github.com/lib/pq/auth/kerberos"
func init() {
pq.RegisterGSSProvider(func() (pq.Gss, error) { return kerberos.NewGSS() })
}
This package is in a separate module so that users who don't need Kerberos
don't have to download unnecessary dependencies.
When imported, additional connection string parameters are supported:
* krbsrvname - GSS (Kerberos) service name when constructing the
SPN (default is `postgres`). This will be combined with the host
to form the full SPN: `krbsrvname/host`.
* krbspn - GSS (Kerberos) SPN. This takes priority over
`krbsrvname` if present.
*/
package pq
......@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"math"
"regexp"
"strconv"
"strings"
"sync"
......@@ -16,6 +17,8 @@ import (
"github.com/lib/pq/oid"
)
var time2400Regex = regexp.MustCompile(`^(24:00(?::00(?:\.0+)?)?)(?:[Z+-].*)?$`)
func binaryEncode(parameterStatus *parameterStatus, x interface{}) []byte {
switch v := x.(type) {
case []byte:
......@@ -202,10 +205,27 @@ func mustParse(f string, typ oid.Oid, s []byte) time.Time {
str[len(str)-3] == ':' {
f += ":00"
}
// Special case for 24:00 time.
// Unfortunately, golang does not parse 24:00 as a proper time.
// In this case, we want to try "round to the next day", to differentiate.
// As such, we find if the 24:00 time matches at the beginning; if so,
// we default it back to 00:00 but add a day later.
var is2400Time bool
switch typ {
case oid.T_timetz, oid.T_time:
if matches := time2400Regex.FindStringSubmatch(str); matches != nil {
// Concatenate timezone information at the back.
str = "00:00:00" + str[len(matches[1]):]
is2400Time = true
}
}
t, err := time.Parse(f, str)
if err != nil {
errorf("decode: %s", err)
}
if is2400Time {
t = t.Add(24 * time.Hour)
}
return t
}
......@@ -487,7 +507,7 @@ func FormatTimestamp(t time.Time) []byte {
b := []byte(t.Format("2006-01-02 15:04:05.999999999Z07:00"))
_, offset := t.Zone()
offset = offset % 60
offset %= 60
if offset != 0 {
// RFC3339Nano already printed the minus sign
if offset < 0 {
......
......@@ -478,13 +478,13 @@ func errRecoverNoErrBadConn(err *error) {
}
}
func (c *conn) errRecover(err *error) {
func (cn *conn) errRecover(err *error) {
e := recover()
switch v := e.(type) {
case nil:
// Do nothing
case runtime.Error:
c.bad = true
cn.bad = true
panic(v)
case *Error:
if v.Fatal() {
......@@ -493,7 +493,7 @@ func (c *conn) errRecover(err *error) {
*err = v
}
case *net.OpError:
c.bad = true
cn.bad = true
*err = v
case error:
if v == io.EOF || v.(error).Error() == "remote error: handshake failure" {
......@@ -503,13 +503,13 @@ func (c *conn) errRecover(err *error) {
}
default:
c.bad = true
cn.bad = true
panic(fmt.Sprintf("unknown error: %#v", e))
}
// Any time we return ErrBadConn, we need to remember it since *Tx doesn't
// mark the connection bad in database/sql.
if *err == driver.ErrBadConn {
c.bad = true
cn.bad = true
}
}
package pq
// NewGSSFunc creates a GSS authentication provider, for use with
// RegisterGSSProvider.
type NewGSSFunc func() (GSS, error)
var newGss NewGSSFunc
// RegisterGSSProvider registers a GSS authentication provider. For example, if
// you need to use Kerberos to authenticate with your server, add this to your
// main package:
//
// import "github.com/lib/pq/auth/kerberos"
//
// func init() {
// pq.RegisterGSSProvider(func() (pq.GSS, error) { return kerberos.NewGSS() })
// }
func RegisterGSSProvider(newGssArg NewGSSFunc) {
newGss = newGssArg
}
// GSS provides GSSAPI authentication (e.g., Kerberos).
type GSS interface {
GetInitToken(host string, service string) ([]byte, error)
GetInitTokenFromSpn(spn string) ([]byte, error)
Continue(inToken []byte) (done bool, outToken []byte, err error)
}
// +build go1.10
package pq
import (
"context"
"database/sql/driver"
)
// NoticeHandler returns the notice handler on the given connection, if any. A
// runtime panic occurs if c is not a pq connection. This is rarely used
// directly, use ConnectorNoticeHandler and ConnectorWithNoticeHandler instead.
func NoticeHandler(c driver.Conn) func(*Error) {
return c.(*conn).noticeHandler
}
// SetNoticeHandler sets the given notice handler on the given connection. A
// runtime panic occurs if c is not a pq connection. A nil handler may be used
// to unset it. This is rarely used directly, use ConnectorNoticeHandler and
// ConnectorWithNoticeHandler instead.
//
// Note: Notice handlers are executed synchronously by pq meaning commands
// won't continue to be processed until the handler returns.
func SetNoticeHandler(c driver.Conn, handler func(*Error)) {
c.(*conn).noticeHandler = handler
}
// NoticeHandlerConnector wraps a regular connector and sets a notice handler
// on it.
type NoticeHandlerConnector struct {
driver.Connector
noticeHandler func(*Error)
}
// Connect calls the underlying connector's connect method and then sets the
// notice handler.
func (n *NoticeHandlerConnector) Connect(ctx context.Context) (driver.Conn, error) {
c, err := n.Connector.Connect(ctx)
if err == nil {
SetNoticeHandler(c, n.noticeHandler)
}
return c, err
}
// ConnectorNoticeHandler returns the currently set notice handler, if any. If
// the given connector is not a result of ConnectorWithNoticeHandler, nil is
// returned.
func ConnectorNoticeHandler(c driver.Connector) func(*Error) {
if c, ok := c.(*NoticeHandlerConnector); ok {
return c.noticeHandler
}
return nil
}
// ConnectorWithNoticeHandler creates or sets the given handler for the given
// connector. If the given connector is a result of calling this function
// previously, it is simply set on the given connector and returned. Otherwise,
// this returns a new connector wrapping the given one and setting the notice
// handler. A nil notice handler may be used to unset it.
//
// The returned connector is intended to be used with database/sql.OpenDB.
//
// Note: Notice handlers are executed synchronously by pq meaning commands
// won't continue to be processed until the handler returns.
func ConnectorWithNoticeHandler(c driver.Connector, handler func(*Error)) *NoticeHandlerConnector {
if c, ok := c.(*NoticeHandlerConnector); ok {
c.noticeHandler = handler
return c
}
return &NoticeHandlerConnector{Connector: c, noticeHandler: handler}
}
......@@ -4,6 +4,8 @@ package pq
// This module contains support for Postgres LISTEN/NOTIFY.
import (
"context"
"database/sql/driver"
"errors"
"fmt"
"sync"
......@@ -29,6 +31,61 @@ func recvNotification(r *readBuf) *Notification {
return &Notification{bePid, channel, extra}
}
// SetNotificationHandler sets the given notification handler on the given
// connection. A runtime panic occurs if c is not a pq connection. A nil handler
// may be used to unset it.
//
// Note: Notification handlers are executed synchronously by pq meaning commands
// won't continue to be processed until the handler returns.
func SetNotificationHandler(c driver.Conn, handler func(*Notification)) {
c.(*conn).notificationHandler = handler
}
// NotificationHandlerConnector wraps a regular connector and sets a notification handler
// on it.
type NotificationHandlerConnector struct {
driver.Connector
notificationHandler func(*Notification)
}
// Connect calls the underlying connector's connect method and then sets the
// notification handler.
func (n *NotificationHandlerConnector) Connect(ctx context.Context) (driver.Conn, error) {
c, err := n.Connector.Connect(ctx)
if err == nil {
SetNotificationHandler(c, n.notificationHandler)
}
return c, err
}
// ConnectorNotificationHandler returns the currently set notification handler, if any. If
// the given connector is not a result of ConnectorWithNotificationHandler, nil is
// returned.
func ConnectorNotificationHandler(c driver.Connector) func(*Notification) {
if c, ok := c.(*NotificationHandlerConnector); ok {
return c.notificationHandler
}
return nil
}
// ConnectorWithNotificationHandler creates or sets the given handler for the given
// connector. If the given connector is a result of calling this function
// previously, it is simply set on the given connector and returned. Otherwise,
// this returns a new connector wrapping the given one and setting the notification
// handler. A nil notification handler may be used to unset it.
//
// The returned connector is intended to be used with database/sql.OpenDB.
//
// Note: Notification handlers are executed synchronously by pq meaning commands
// won't continue to be processed until the handler returns.
func ConnectorWithNotificationHandler(c driver.Connector, handler func(*Notification)) *NotificationHandlerConnector {
if c, ok := c.(*NotificationHandlerConnector); ok {
c.notificationHandler = handler
return c
}
return &NotificationHandlerConnector{Connector: c, notificationHandler: handler}
}
const (
connStateIdle int32 = iota
connStateExpectResponse
......@@ -174,8 +231,12 @@ func (l *ListenerConn) listenerConnLoop() (err error) {
}
l.replyChan <- message{t, nil}
case 'N', 'S':
case 'S':
// ignore
case 'N':
if n := l.cn.noticeHandler; n != nil {
n(parseError(r))
}
default:
return fmt.Errorf("unexpected message %q from server in listenerConnLoop", t)
}
......
......@@ -22,7 +22,7 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Pacakage scram implements a SCRAM-{SHA-1,etc} client per RFC5802.
// Package scram implements a SCRAM-{SHA-1,etc} client per RFC5802.
//
// http://tools.ietf.org/html/rfc5802
//
......@@ -94,7 +94,7 @@ func (c *Client) Out() []byte {
return c.out.Bytes()
}
// Err returns the error that ocurred, or nil if there were no errors.
// Err returns the error that occurred, or nil if there were no errors.
func (c *Client) Err() error {
return c.err
}
......
// Package pq is a pure Go Postgres driver for the database/sql package.
// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris rumprun
// +build aix darwin dragonfly freebsd linux nacl netbsd openbsd plan9 solaris rumprun
package pq
......
coverage:
status:
project: off
patch: off
language: go
os:
- linux
- osx
addons:
apt:
update: true
go:
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12.x
- 1.13.x
- master
before_install:
- |
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew update
fi
- go get github.com/smartystreets/goconvey
- go get github.com/mattn/goveralls
- go get golang.org/x/tools/cmd/cover
script:
- $HOME/gopath/bin/goveralls -repotoken 3qJVUE0iQwqnCbmNcDsjYu1nh4J4KIFXx
- go test -race -v . -tags ""
- go test -race -v . -tags "libsqlite3"
- go test -race -v . -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify"
- go test -race -v . -tags "sqlite_vacuum_full"
......@@ -2,16 +2,20 @@ go-sqlite3
==========
[![GoDoc Reference](https://godoc.org/github.com/mattn/go-sqlite3?status.svg)](http://godoc.org/github.com/mattn/go-sqlite3)
[![Build Status](https://travis-ci.org/mattn/go-sqlite3.svg?branch=master)](https://travis-ci.org/mattn/go-sqlite3)
[![GitHub Actions](https://github.com/mattn/go-sqlite3/workflows/Go/badge.svg)](https://github.com/mattn/go-sqlite3/actions?query=workflow%3AGo)
[![Financial Contributors on Open Collective](https://opencollective.com/mattn-go-sqlite3/all/badge.svg?label=financial+contributors)](https://opencollective.com/mattn-go-sqlite3)
[![Coverage Status](https://coveralls.io/repos/mattn/go-sqlite3/badge.svg?branch=master)](https://coveralls.io/r/mattn/go-sqlite3?branch=master)
[![codecov](https://codecov.io/gh/mattn/go-sqlite3/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-sqlite3)
[![Go Report Card](https://goreportcard.com/badge/github.com/mattn/go-sqlite3)](https://goreportcard.com/report/github.com/mattn/go-sqlite3)
Latest stable version is v1.14 or later not v2.
~~**NOTE:** The increase to v2 was an accident. There were no major changes or features.~~
# Description
sqlite3 driver conforming to the built-in database/sql interface
Supported Golang version: See .travis.yml
Supported Golang version: See .github/workflows/go.yaml
[This package follows the official Golang Release Policy.](https://golang.org/doc/devel/release.html#policy)
......@@ -210,9 +214,15 @@ This library can be cross-compiled.
In some cases you are required to the `CC` environment variable with the cross compiler.
Additional information:
- [#491](https://github.com/mattn/go-sqlite3/issues/491)
- [#560](https://github.com/mattn/go-sqlite3/issues/560)
## Cross Compiling from MAC OSX
The simplest way to cross compile from OSX is to use [xgo](https://github.com/karalabe/xgo).
Steps:
- Install [xgo](https://github.com/karalabe/xgo) (`go get github.com/karalabe/xgo`).
- Ensure that your project is within your `GOPATH`.
- Run `xgo local/path/to/project`.
Please refer to the project's [README](https://github.com/karalabe/xgo/blob/master/README.md) for further information.
# Google Cloud Platform
......@@ -451,6 +461,16 @@ If you want your own extension to be listed here or you want to add a reference
Spatialite is available as an extension to SQLite, and can be used in combination with this repository.
For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatialite).
## extension-functions.c from SQLite3 Contrib
extension-functions.c is available as an extension to SQLite, and provides the following functions:
- Math: acos, asin, atan, atn2, atan2, acosh, asinh, atanh, difference, degrees, radians, cos, sin, tan, cot, cosh, sinh, tanh, coth, exp, log, log10, power, sign, sqrt, square, ceil, floor, pi.
- String: replicate, charindex, leftstr, rightstr, ltrim, rtrim, trim, replace, reverse, proper, padl, padr, padc, strfilter.
- Aggregate: stdev, variance, mode, median, lower_quartile, upper_quartile
For an example see [dinedal/go-sqlite3-extension-functions](https://github.com/dinedal/go-sqlite3-extension-functions).
# FAQ
- Getting insert error while query is opened.
......
......@@ -35,56 +35,55 @@ import (
//export callbackTrampoline
func callbackTrampoline(ctx *C.sqlite3_context, argc int, argv **C.sqlite3_value) {
args := (*[(math.MaxInt32 - 1) / unsafe.Sizeof((*C.sqlite3_value)(nil))]*C.sqlite3_value)(unsafe.Pointer(argv))[:argc:argc]
fi := lookupHandle(uintptr(C.sqlite3_user_data(ctx))).(*functionInfo)
fi := lookupHandle(C.sqlite3_user_data(ctx)).(*functionInfo)
fi.Call(ctx, args)
}
//export stepTrampoline
func stepTrampoline(ctx *C.sqlite3_context, argc C.int, argv **C.sqlite3_value) {
args := (*[(math.MaxInt32 - 1) / unsafe.Sizeof((*C.sqlite3_value)(nil))]*C.sqlite3_value)(unsafe.Pointer(argv))[:int(argc):int(argc)]
ai := lookupHandle(uintptr(C.sqlite3_user_data(ctx))).(*aggInfo)
ai := lookupHandle(C.sqlite3_user_data(ctx)).(*aggInfo)
ai.Step(ctx, args)
}
//export doneTrampoline
func doneTrampoline(ctx *C.sqlite3_context) {
handle := uintptr(C.sqlite3_user_data(ctx))
ai := lookupHandle(handle).(*aggInfo)
ai := lookupHandle(C.sqlite3_user_data(ctx)).(*aggInfo)
ai.Done(ctx)
}
//export compareTrampoline
func compareTrampoline(handlePtr uintptr, la C.int, a *C.char, lb C.int, b *C.char) C.int {
func compareTrampoline(handlePtr unsafe.Pointer, la C.int, a *C.char, lb C.int, b *C.char) C.int {
cmp := lookupHandle(handlePtr).(func(string, string) int)
return C.int(cmp(C.GoStringN(a, la), C.GoStringN(b, lb)))
}
//export commitHookTrampoline
func commitHookTrampoline(handle uintptr) int {
func commitHookTrampoline(handle unsafe.Pointer) int {
callback := lookupHandle(handle).(func() int)
return callback()
}
//export rollbackHookTrampoline
func rollbackHookTrampoline(handle uintptr) {
func rollbackHookTrampoline(handle unsafe.Pointer) {
callback := lookupHandle(handle).(func())
callback()
}
//export updateHookTrampoline
func updateHookTrampoline(handle uintptr, op int, db *C.char, table *C.char, rowid int64) {
func updateHookTrampoline(handle unsafe.Pointer, op int, db *C.char, table *C.char, rowid int64) {
callback := lookupHandle(handle).(func(int, string, string, int64))
callback(op, C.GoString(db), C.GoString(table), rowid)
}
//export authorizerTrampoline
func authorizerTrampoline(handle uintptr, op int, arg1 *C.char, arg2 *C.char, arg3 *C.char) int {
func authorizerTrampoline(handle unsafe.Pointer, op int, arg1 *C.char, arg2 *C.char, arg3 *C.char) int {
callback := lookupHandle(handle).(func(int, string, string, string) int)
return callback(op, C.GoString(arg1), C.GoString(arg2), C.GoString(arg3))
}
//export preUpdateHookTrampoline
func preUpdateHookTrampoline(handle uintptr, dbHandle uintptr, op int, db *C.char, table *C.char, oldrowid int64, newrowid int64) {
func preUpdateHookTrampoline(handle unsafe.Pointer, dbHandle uintptr, op int, db *C.char, table *C.char, oldrowid int64, newrowid int64) {
hval := lookupHandleVal(handle)
data := SQLitePreUpdateData{
Conn: hval.db,
......@@ -105,33 +104,27 @@ type handleVal struct {
}
var handleLock sync.Mutex
var handleVals = make(map[uintptr]handleVal)
var handleIndex uintptr = 100
var handleVals = make(map[unsafe.Pointer]handleVal)
func newHandle(db *SQLiteConn, v interface{}) uintptr {
func newHandle(db *SQLiteConn, v interface{}) unsafe.Pointer {
handleLock.Lock()
defer handleLock.Unlock()
i := handleIndex
handleIndex++
handleVals[i] = handleVal{db, v}
return i
val := handleVal{db: db, val: v}
var p unsafe.Pointer = C.malloc(C.size_t(1))
if p == nil {
panic("can't allocate 'cgo-pointer hack index pointer': ptr == nil")
}
handleVals[p] = val
return p
}
func lookupHandleVal(handle uintptr) handleVal {
func lookupHandleVal(handle unsafe.Pointer) handleVal {
handleLock.Lock()
defer handleLock.Unlock()
r, ok := handleVals[handle]
if !ok {
if handle >= 100 && handle < handleIndex {
panic("deleted handle")
} else {
panic("invalid handle")
}
}
return r
return handleVals[handle]
}
func lookupHandle(handle uintptr) interface{} {
func lookupHandle(handle unsafe.Pointer) interface{} {
return lookupHandleVal(handle).val
}
......@@ -141,6 +134,7 @@ func deleteHandles(db *SQLiteConn) {
for handle, val := range handleVals {
if val.db == db {
delete(handleVals, handle)
C.free(handle)
}
}
}
......
......@@ -5,7 +5,15 @@
package sqlite3
/*
#ifndef USE_LIBSQLITE3
#include <sqlite3-binding.h>
#else
#include <sqlite3.h>
#endif
*/
import "C"
import "syscall"
// ErrNo inherit errno.
type ErrNo int
......@@ -20,6 +28,7 @@ type ErrNoExtended int
type Error struct {
Code ErrNo /* The error code returned by SQLite */
ExtendedCode ErrNoExtended /* The extended error code returned by SQLite */
SystemErrno syscall.Errno /* The system errno returned by the OS through SQLite, if applicable */
err string /* The error string returned by sqlite3_errmsg(),
this usually contains more specific details. */
}
......@@ -72,10 +81,16 @@ func (err ErrNoExtended) Error() string {
}
func (err Error) Error() string {
var str string
if err.err != "" {
return err.err
str = err.err
} else {
str = C.GoString(C.sqlite3_errstr(C.int(err.Code)))
}
return errorString(err)
if err.SystemErrno != 0 {
str += ": " + err.SystemErrno.Error()
}
return str
}
// result codes from http://www.sqlite.org/c3ref/c_abort_rollback.html
......
module github.com/mattn/go-sqlite3
go 1.10
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -14,5 +14,6 @@ package sqlite3
#cgo darwin CFLAGS: -I/usr/local/opt/sqlite/include
#cgo openbsd LDFLAGS: -lsqlite3
#cgo solaris LDFLAGS: -lsqlite3
#cgo windows LDFLAGS: -lsqlite3
*/
import "C"
......@@ -28,12 +28,9 @@ func (c *SQLiteConn) loadExtensions(extensions []string) error {
}
for _, extension := range extensions {
cext := C.CString(extension)
defer C.free(unsafe.Pointer(cext))
rv = C.sqlite3_load_extension(c.db, cext, nil, nil)
if rv != C.SQLITE_OK {
if err := c.loadExtension(extension, nil); err != nil {
C.sqlite3_enable_load_extension(c.db, 0)
return errors.New(C.GoString(C.sqlite3_errmsg(c.db)))
return err
}
}
......@@ -41,6 +38,7 @@ func (c *SQLiteConn) loadExtensions(extensions []string) error {
if rv != C.SQLITE_OK {
return errors.New(C.GoString(C.sqlite3_errmsg(c.db)))
}
return nil
}
......@@ -51,19 +49,35 @@ func (c *SQLiteConn) LoadExtension(lib string, entry string) error {
return errors.New(C.GoString(C.sqlite3_errmsg(c.db)))
}
clib := C.CString(lib)
defer C.free(unsafe.Pointer(clib))
centry := C.CString(entry)
defer C.free(unsafe.Pointer(centry))
if err := c.loadExtension(lib, &entry); err != nil {
C.sqlite3_enable_load_extension(c.db, 0)
return err
}
rv = C.sqlite3_load_extension(c.db, clib, centry, nil)
rv = C.sqlite3_enable_load_extension(c.db, 0)
if rv != C.SQLITE_OK {
return errors.New(C.GoString(C.sqlite3_errmsg(c.db)))
}
rv = C.sqlite3_enable_load_extension(c.db, 0)
return nil
}
func (c *SQLiteConn) loadExtension(lib string, entry *string) error {
clib := C.CString(lib)
defer C.free(unsafe.Pointer(clib))
var centry *C.char
if entry != nil {
centry = C.CString(*entry)
defer C.free(unsafe.Pointer(centry))
}
var errMsg *C.char
defer C.sqlite3_free(unsafe.Pointer(errMsg))
rv := C.sqlite3_load_extension(c.db, clib, centry, &errMsg)
if rv != C.SQLITE_OK {
return errors.New(C.GoString(C.sqlite3_errmsg(c.db)))
return errors.New(C.GoString(errMsg))
}
return nil
......
......@@ -288,10 +288,13 @@ type InfoOrderBy struct {
}
func constraints(info *C.sqlite3_index_info) []InfoConstraint {
l := info.nConstraint
slice := (*[1 << 30]C.struct_sqlite3_index_constraint)(unsafe.Pointer(info.aConstraint))[:l:l]
slice := *(*[]C.struct_sqlite3_index_constraint)(unsafe.Pointer(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(info.aConstraint)),
Len: int(info.nConstraint),
Cap: int(info.nConstraint),
}))
cst := make([]InfoConstraint, 0, l)
cst := make([]InfoConstraint, 0, len(slice))
for _, c := range slice {
var usable bool
if c.usable > 0 {
......@@ -307,10 +310,13 @@ func constraints(info *C.sqlite3_index_info) []InfoConstraint {
}
func orderBys(info *C.sqlite3_index_info) []InfoOrderBy {
l := info.nOrderBy
slice := (*[1 << 30]C.struct_sqlite3_index_orderby)(unsafe.Pointer(info.aOrderBy))[:l:l]
slice := *(*[]C.struct_sqlite3_index_orderby)(unsafe.Pointer(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(info.aOrderBy)),
Len: int(info.nOrderBy),
Cap: int(info.nOrderBy),
}))
ob := make([]InfoOrderBy, 0, l)
ob := make([]InfoOrderBy, 0, len(slice))
for _, c := range slice {
var desc bool
if c.desc > 0 {
......@@ -347,7 +353,7 @@ func mPrintf(format, arg string) *C.char {
//export goMInit
func goMInit(db, pClientData unsafe.Pointer, argc C.int, argv **C.char, pzErr **C.char, isCreate C.int) C.uintptr_t {
m := lookupHandle(uintptr(pClientData)).(*sqliteModule)
m := lookupHandle(pClientData).(*sqliteModule)
if m.c.db != (*C.sqlite3)(db) {
*pzErr = mPrintf("%s", "Inconsistent db handles")
return 0
......@@ -373,12 +379,12 @@ func goMInit(db, pClientData unsafe.Pointer, argc C.int, argv **C.char, pzErr **
}
vt := sqliteVTab{m, vTab}
*pzErr = nil
return C.uintptr_t(newHandle(m.c, &vt))
return C.uintptr_t(uintptr(newHandle(m.c, &vt)))
}
//export goVRelease
func goVRelease(pVTab unsafe.Pointer, isDestroy C.int) *C.char {
vt := lookupHandle(uintptr(pVTab)).(*sqliteVTab)
vt := lookupHandle(pVTab).(*sqliteVTab)
var err error
if isDestroy == 1 {
err = vt.vTab.Destroy()
......@@ -393,7 +399,7 @@ func goVRelease(pVTab unsafe.Pointer, isDestroy C.int) *C.char {
//export goVOpen
func goVOpen(pVTab unsafe.Pointer, pzErr **C.char) C.uintptr_t {
vt := lookupHandle(uintptr(pVTab)).(*sqliteVTab)
vt := lookupHandle(pVTab).(*sqliteVTab)
vTabCursor, err := vt.vTab.Open()
if err != nil {
*pzErr = mPrintf("%s", err.Error())
......@@ -401,12 +407,12 @@ func goVOpen(pVTab unsafe.Pointer, pzErr **C.char) C.uintptr_t {
}
vtc := sqliteVTabCursor{vt, vTabCursor}
*pzErr = nil
return C.uintptr_t(newHandle(vt.module.c, &vtc))
return C.uintptr_t(uintptr(newHandle(vt.module.c, &vtc)))
}
//export goVBestIndex
func goVBestIndex(pVTab unsafe.Pointer, icp unsafe.Pointer) *C.char {
vt := lookupHandle(uintptr(pVTab)).(*sqliteVTab)
vt := lookupHandle(pVTab).(*sqliteVTab)
info := (*C.sqlite3_index_info)(icp)
csts := constraints(info)
res, err := vt.vTab.BestIndex(csts, orderBys(info))
......@@ -418,13 +424,17 @@ func goVBestIndex(pVTab unsafe.Pointer, icp unsafe.Pointer) *C.char {
}
// Get a pointer to constraint_usage struct so we can update in place.
l := info.nConstraint
s := (*[1 << 30]C.struct_sqlite3_index_constraint_usage)(unsafe.Pointer(info.aConstraintUsage))[:l:l]
slice := *(*[]C.struct_sqlite3_index_constraint_usage)(unsafe.Pointer(&reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(info.aConstraintUsage)),
Len: int(info.nConstraint),
Cap: int(info.nConstraint),
}))
index := 1
for i := C.int(0); i < info.nConstraint; i++ {
for i := range slice {
if res.Used[i] {
s[i].argvIndex = C.int(index)
s[i].omit = C.uchar(1)
slice[i].argvIndex = C.int(index)
slice[i].omit = C.uchar(1)
index++
}
}
......@@ -445,7 +455,7 @@ func goVBestIndex(pVTab unsafe.Pointer, icp unsafe.Pointer) *C.char {
//export goVClose
func goVClose(pCursor unsafe.Pointer) *C.char {
vtc := lookupHandle(uintptr(pCursor)).(*sqliteVTabCursor)
vtc := lookupHandle(pCursor).(*sqliteVTabCursor)
err := vtc.vTabCursor.Close()
if err != nil {
return mPrintf("%s", err.Error())
......@@ -455,13 +465,13 @@ func goVClose(pCursor unsafe.Pointer) *C.char {
//export goMDestroy
func goMDestroy(pClientData unsafe.Pointer) {
m := lookupHandle(uintptr(pClientData)).(*sqliteModule)
m := lookupHandle(pClientData).(*sqliteModule)
m.module.DestroyModule()
}
//export goVFilter
func goVFilter(pCursor unsafe.Pointer, idxNum C.int, idxName *C.char, argc C.int, argv **C.sqlite3_value) *C.char {
vtc := lookupHandle(uintptr(pCursor)).(*sqliteVTabCursor)
vtc := lookupHandle(pCursor).(*sqliteVTabCursor)
args := (*[(math.MaxInt32 - 1) / unsafe.Sizeof((*C.sqlite3_value)(nil))]*C.sqlite3_value)(unsafe.Pointer(argv))[:argc:argc]
vals := make([]interface{}, 0, argc)
for _, v := range args {
......@@ -480,7 +490,7 @@ func goVFilter(pCursor unsafe.Pointer, idxNum C.int, idxName *C.char, argc C.int
//export goVNext
func goVNext(pCursor unsafe.Pointer) *C.char {
vtc := lookupHandle(uintptr(pCursor)).(*sqliteVTabCursor)
vtc := lookupHandle(pCursor).(*sqliteVTabCursor)
err := vtc.vTabCursor.Next()
if err != nil {
return mPrintf("%s", err.Error())
......@@ -490,7 +500,7 @@ func goVNext(pCursor unsafe.Pointer) *C.char {
//export goVEof
func goVEof(pCursor unsafe.Pointer) C.int {
vtc := lookupHandle(uintptr(pCursor)).(*sqliteVTabCursor)
vtc := lookupHandle(pCursor).(*sqliteVTabCursor)
err := vtc.vTabCursor.EOF()
if err {
return 1
......@@ -500,7 +510,7 @@ func goVEof(pCursor unsafe.Pointer) C.int {
//export goVColumn
func goVColumn(pCursor, cp unsafe.Pointer, col C.int) *C.char {
vtc := lookupHandle(uintptr(pCursor)).(*sqliteVTabCursor)
vtc := lookupHandle(pCursor).(*sqliteVTabCursor)
c := (*SQLiteContext)(cp)
err := vtc.vTabCursor.Column(c, int(col))
if err != nil {
......@@ -511,7 +521,7 @@ func goVColumn(pCursor, cp unsafe.Pointer, col C.int) *C.char {
//export goVRowid
func goVRowid(pCursor unsafe.Pointer, pRowid *C.sqlite3_int64) *C.char {
vtc := lookupHandle(uintptr(pCursor)).(*sqliteVTabCursor)
vtc := lookupHandle(pCursor).(*sqliteVTabCursor)
rowid, err := vtc.vTabCursor.Rowid()
if err != nil {
return mPrintf("%s", err.Error())
......@@ -522,7 +532,7 @@ func goVRowid(pCursor unsafe.Pointer, pRowid *C.sqlite3_int64) *C.char {
//export goVUpdate
func goVUpdate(pVTab unsafe.Pointer, argc C.int, argv **C.sqlite3_value, pRowid *C.sqlite3_int64) *C.char {
vt := lookupHandle(uintptr(pVTab)).(*sqliteVTab)
vt := lookupHandle(pVTab).(*sqliteVTab)
var tname string
if n, ok := vt.vTab.(interface {
......@@ -642,7 +652,7 @@ func (c *SQLiteConn) CreateModule(moduleName string, module Module) error {
mname := C.CString(moduleName)
defer C.free(unsafe.Pointer(mname))
udm := sqliteModule{c, moduleName, module}
rv := C._sqlite3_create_module(c.db, mname, C.uintptr_t(newHandle(c, &udm)))
rv := C._sqlite3_create_module(c.db, mname, C.uintptr_t(uintptr(newHandle(c, &udm))))
if rv != C.SQLITE_OK {
return c.lastError()
}
......
......@@ -215,7 +215,6 @@ func addTraceMapping(connHandle uintptr, traceConf TraceConfig) {
traceConf, connHandle, oldEntryCopy.config))
}
traceMap[connHandle] = traceMapEntry{config: traceConf}
fmt.Printf("Added trace config %v: handle 0x%x.\n", traceConf, connHandle)
}
func lookupTraceMapping(connHandle uintptr) (TraceConfig, bool) {
......@@ -234,7 +233,6 @@ func popTraceMapping(connHandle uintptr) (TraceConfig, bool) {
entryCopy, found := traceMap[connHandle]
if found {
delete(traceMap, connHandle)
fmt.Printf("Pop handle 0x%x: deleted trace config %v.\n", connHandle, entryCopy.config)
}
return entryCopy.config, found
}
......
......@@ -325,6 +325,17 @@ struct sqlite3_api_routines {
int (*value_frombind)(sqlite3_value*);
/* Version 3.30.0 and later */
int (*drop_modules)(sqlite3*,const char**);
/* Version 3.31.0 and later */
sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64);
const char *(*uri_key)(const char*,int);
const char *(*filename_database)(const char*);
const char *(*filename_journal)(const char*);
const char *(*filename_wal)(const char*);
/* Version 3.32.0 and later */
char *(*create_filename)(const char*,const char*,const char*,
int,const char**);
void (*free_filename)(char*);
sqlite3_file *(*database_file_object)(const char*);
};
/*
......@@ -615,10 +626,20 @@ typedef int (*sqlite3_loadext_entry)(
/* Version 3.26.0 and later */
#define sqlite3_normalized_sql sqlite3_api->normalized_sql
/* Version 3.28.0 and later */
#define sqlite3_stmt_isexplain sqlite3_api->isexplain
#define sqlite3_value_frombind sqlite3_api->frombind
#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain
#define sqlite3_value_frombind sqlite3_api->value_frombind
/* Version 3.30.0 and later */
#define sqlite3_drop_modules sqlite3_api->drop_modules
/* Version 3.31.0 and later */
#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64
#define sqlite3_uri_key sqlite3_api->uri_key
#define sqlite3_filename_database sqlite3_api->filename_database
#define sqlite3_filename_journal sqlite3_api->filename_journal
#define sqlite3_filename_wal sqlite3_api->filename_wal
/* Version 3.32.0 and later */
#define sqlite3_create_filename sqlite3_api->create_filename
#define sqlite3_free_filename sqlite3_api->free_filename
#define sqlite3_database_file_object sqlite3_api->database_file_object
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
......
......@@ -13,14 +13,25 @@ import (
"errors"
)
var errorMsg = errors.New("Binary was compiled with 'CGO_ENABLED=0', go-sqlite3 requires cgo to work. This is a stub")
func init() {
sql.Register("sqlite3", &SQLiteDriverMock{})
sql.Register("sqlite3", &SQLiteDriver{})
}
type SQLiteDriverMock struct{}
var errorMsg = errors.New("Binary was compiled with 'CGO_ENABLED=0', go-sqlite3 requires cgo to work. This is a stub")
type (
SQLiteDriver struct {
Extensions []string
ConnectHook func(*SQLiteConn) error
}
SQLiteConn struct{}
)
func (SQLiteDriverMock) Open(s string) (driver.Conn, error) {
return nil, errorMsg
}
func (SQLiteDriver) Open(s string) (driver.Conn, error) { return nil, errorMsg }
func (c *SQLiteConn) RegisterAggregator(string, interface{}, bool) error { return errorMsg }
func (c *SQLiteConn) RegisterAuthorizer(func(int, string, string, string) int) {}
func (c *SQLiteConn) RegisterCollation(string, func(string, string) int) error { return errorMsg }
func (c *SQLiteConn) RegisterCommitHook(func() int) {}
func (c *SQLiteConn) RegisterFunc(string, interface{}, bool) error { return errorMsg }
func (c *SQLiteConn) RegisterRollbackHook(func()) {}
func (c *SQLiteConn) RegisterUpdateHook(func(int, string, string, int64)) {}
......@@ -15,6 +15,7 @@ import (
"github.com/canonical/go-dqlite/client"
"github.com/canonical/go-dqlite/driver"
"github.com/pkg/errors"
"github.com/rancher/kine/pkg/drivers/generic"
"github.com/rancher/kine/pkg/drivers/sqlite"
"github.com/rancher/kine/pkg/server"
"github.com/sirupsen/logrus"
......@@ -66,7 +67,7 @@ outer:
return nil
}
func New(ctx context.Context, datasourceName string) (server.Backend, error) {
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
opts, err := parseOpts(datasourceName)
if err != nil {
return nil, err
......@@ -95,7 +96,7 @@ func New(ctx context.Context, datasourceName string) (server.Backend, error) {
}
sql.Register("dqlite", d)
backend, generic, err := sqlite.NewVariant(ctx, "dqlite", opts.dsn)
backend, generic, err := sqlite.NewVariant(ctx, "dqlite", opts.dsn, connPoolConfig)
if err != nil {
return nil, errors.Wrap(err, "sqlite client")
}
......
......@@ -4,11 +4,12 @@ package dqlite
import (
"context"
"fmt"
"errors"
"github.com/rancher/kine/pkg/drivers/generic"
"github.com/rancher/kine/pkg/server"
)
func New(ctx context.Context, datasourceName string) (server.Backend, error) {
return nil, fmt.Errorf("dqlite is not support, compile with \"-tags dqlite\"")
func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
return nil, errors.New(`this binary is built without dqlite support, compile with "-tags dqlite"`)
}
......@@ -15,34 +15,37 @@ import (
"github.com/sirupsen/logrus"
)
const (
defaultMaxIdleConns = 2 // copied from database/sql
)
var (
columns = "kv.id as theid, kv.name, kv.created, kv.deleted, kv.create_revision, kv.prev_revision, kv.lease, kv.value, kv.old_value"
columns = "kv.id AS theid, kv.name, kv.created, kv.deleted, kv.create_revision, kv.prev_revision, kv.lease, kv.value, kv.old_value"
revSQL = `
SELECT rkv.id
FROM kine rkv
ORDER BY rkv.id
DESC LIMIT 1`
SELECT MAX(rkv.id) AS id
FROM kine AS rkv`
compactRevSQL = `
SELECT crkv.prev_revision
FROM kine crkv
WHERE crkv.name = 'compact_rev_key'
ORDER BY crkv.id DESC LIMIT 1`
SELECT MAX(crkv.prev_revision) AS prev_revision
FROM kine AS crkv
WHERE crkv.name = 'compact_rev_key'`
idOfKey = `
AND mkv.id <= ? AND mkv.id > (
SELECT ikv.id
FROM kine ikv
AND
mkv.id <= ? AND
mkv.id > (
SELECT MAX(ikv.id) AS id
FROM kine AS ikv
WHERE
ikv.name = ? AND
ikv.id <= ?
ORDER BY ikv.id DESC LIMIT 1)`
ikv.id <= ?)`
listSQL = fmt.Sprintf(`SELECT (%s), (%s), %s
FROM kine kv
listSQL = fmt.Sprintf(`
SELECT (%s), (%s), %s
FROM kine AS kv
JOIN (
SELECT MAX(mkv.id) as id
FROM kine mkv
SELECT MAX(mkv.id) AS id
FROM kine AS mkv
WHERE
mkv.name LIKE ?
%%s
......@@ -64,6 +67,12 @@ func (s Stripped) String() string {
type ErrRetry func(error) bool
type TranslateErr func(error) error
type ConnectionPoolConfig struct {
MaxIdle int // zero means defaultMaxIdleConns; negative means 0
MaxOpen int // <= 0 means unlimited
MaxLifetime time.Duration // maximum amount of time a connection may be reused
}
type Generic struct {
sync.Mutex
......@@ -78,6 +87,7 @@ type Generic struct {
CountSQL string
AfterSQL string
DeleteSQL string
CompactSQL string
UpdateCompactSQL string
InsertSQL string
FillSQL string
......@@ -128,6 +138,20 @@ func (d *Generic) Migrate(ctx context.Context) {
}
}
func configureConnectionPooling(connPoolConfig ConnectionPoolConfig, db *sql.DB, driverName string) {
// behavior copied from database/sql - zero means defaultMaxIdleConns; negative means 0
if connPoolConfig.MaxIdle < 0 {
connPoolConfig.MaxIdle = 0
} else if connPoolConfig.MaxIdle == 0 {
connPoolConfig.MaxIdle = defaultMaxIdleConns
}
logrus.Infof("Configuring %s database connection pooling: maxIdleConns=%d, maxOpenConns=%d, connMaxLifetime=%s", driverName, connPoolConfig.MaxIdle, connPoolConfig.MaxOpen, connPoolConfig.MaxLifetime)
db.SetMaxIdleConns(connPoolConfig.MaxIdle)
db.SetMaxOpenConns(connPoolConfig.MaxOpen)
db.SetConnMaxLifetime(connPoolConfig.MaxLifetime)
}
func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
db, err := sql.Open(driverName, dataSourceName)
if err != nil {
......@@ -144,7 +168,7 @@ func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
return db, nil
}
func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter string, numbered bool) (*Generic, error) {
func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool) (*Generic, error) {
var (
db *sql.DB
err error
......@@ -164,13 +188,15 @@ func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter
}
}
configureConnectionPooling(connPoolConfig, db, driverName)
return &Generic{
DB: db,
GetRevisionSQL: q(fmt.Sprintf(`
SELECT
0, 0, %s
FROM kine kv
FROM kine AS kv
WHERE kv.id = ?`, columns), paramCharacter, numbered),
GetCurrentSQL: q(fmt.Sprintf(listSQL, ""), paramCharacter, numbered),
......@@ -185,15 +211,15 @@ func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter
AfterSQL: q(fmt.Sprintf(`
SELECT (%s), (%s), %s
FROM kine kv
FROM kine AS kv
WHERE
kv.name LIKE ? AND
kv.id > ?
ORDER BY kv.id ASC`, revSQL, compactRevSQL, columns), paramCharacter, numbered),
DeleteSQL: q(`
DELETE FROM kine
WHERE id = ?`, paramCharacter, numbered),
DELETE FROM kine AS kv
WHERE kv.id = ?`, paramCharacter, numbered),
UpdateCompactSQL: q(`
UPDATE kine
......@@ -229,11 +255,7 @@ func (d *Generic) execute(ctx context.Context, sql string, args ...interface{})
wait := strategy.Backoff(backoff.Linear(100 + time.Millisecond))
for i := uint(0); i < 20; i++ {
if i > 2 {
logrus.Debugf("EXEC (try: %d) %v : %s", i, args, Stripped(sql))
} else {
logrus.Tracef("EXEC (try: %d) %v : %s", i, args, Stripped(sql))
}
logrus.Tracef("EXEC (try: %d) %v : %s", i, args, Stripped(sql))
result, err = d.DB.ExecContext(ctx, sql, args...)
if err != nil && d.Retry != nil && d.Retry(err) {
wait(i)
......@@ -255,15 +277,26 @@ func (d *Generic) GetCompactRevision(ctx context.Context) (int64, error) {
}
func (d *Generic) SetCompactRevision(ctx context.Context, revision int64) error {
logrus.Tracef("SETCOMPACTREVISION %v", revision)
_, err := d.execute(ctx, d.UpdateCompactSQL, revision)
return err
}
func (d *Generic) Compact(ctx context.Context, revision int64) (int64, error) {
logrus.Tracef("COMPACT %v", revision)
res, err := d.execute(ctx, d.CompactSQL, revision, revision)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func (d *Generic) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) {
return d.query(ctx, d.GetRevisionSQL, revision)
}
func (d *Generic) DeleteRevision(ctx context.Context, revision int64) error {
logrus.Tracef("DELETEREVISION %v", revision)
_, err := d.execute(ctx, d.DeleteSQL, revision)
return err
}
......
package generic
import (
"context"
"database/sql"
"github.com/sirupsen/logrus"
)
type Tx struct {
x *sql.Tx
d *Generic
}
func (d *Generic) BeginTx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) {
logrus.Tracef("TX BEGIN")
x, err := d.DB.BeginTx(ctx, opts)
if err != nil {
return nil, err
}
return &Tx{
x: x,
d: d,
}, nil
}
func (t *Tx) Commit() error {
logrus.Tracef("TX COMMIT")
return t.x.Commit()
}
func (t *Tx) MustCommit() {
if err := t.Commit(); err != nil {
logrus.Fatalf("Transaction commit failed: %v", err)
}
}
func (t *Tx) Rollback() error {
logrus.Tracef("TX ROLLBACK")
return t.x.Rollback()
}
func (t *Tx) MustRollback() {
if err := t.Rollback(); err != nil {
if err != sql.ErrTxDone {
logrus.Fatalf("Transaction rollback failed: %v", err)
}
}
}
func (t *Tx) GetCompactRevision(ctx context.Context) (int64, error) {
var id int64
row := t.queryRow(ctx, compactRevSQL)
err := row.Scan(&id)
if err == sql.ErrNoRows {
return 0, nil
}
return id, err
}
func (t *Tx) SetCompactRevision(ctx context.Context, revision int64) error {
logrus.Tracef("TX SETCOMPACTREVISION %v", revision)
_, err := t.execute(ctx, t.d.UpdateCompactSQL, revision)
return err
}
func (t *Tx) Compact(ctx context.Context, revision int64) (int64, error) {
logrus.Tracef("TX COMPACT %v", revision)
res, err := t.execute(ctx, t.d.CompactSQL, revision, revision)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func (t *Tx) GetRevision(ctx context.Context, revision int64) (*sql.Rows, error) {
return t.query(ctx, t.d.GetRevisionSQL, revision)
}
func (t *Tx) DeleteRevision(ctx context.Context, revision int64) error {
logrus.Tracef("TX DELETEREVISION %v", revision)
_, err := t.execute(ctx, t.d.DeleteSQL, revision)
return err
}
func (t *Tx) CurrentRevision(ctx context.Context) (int64, error) {
var id int64
row := t.queryRow(ctx, revSQL)
err := row.Scan(&id)
if err == sql.ErrNoRows {
return 0, nil
}
return id, err
}
func (t *Tx) query(ctx context.Context, sql string, args ...interface{}) (*sql.Rows, error) {
logrus.Tracef("TX QUERY %v : %s", args, Stripped(sql))
return t.x.QueryContext(ctx, sql, args...)
}
func (t *Tx) queryRow(ctx context.Context, sql string, args ...interface{}) *sql.Row {
logrus.Tracef("TX QUERY ROW %v : %s", args, Stripped(sql))
return t.x.QueryRowContext(ctx, sql, args...)
}
func (t *Tx) execute(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error) {
logrus.Tracef("TX EXEC %v : %s", args, Stripped(sql))
return t.x.ExecContext(ctx, sql, args...)
}
......@@ -11,6 +11,7 @@ import (
"github.com/rancher/kine/pkg/logstructured/sqllog"
"github.com/rancher/kine/pkg/server"
"github.com/rancher/kine/pkg/tls"
"github.com/sirupsen/logrus"
)
const (
......@@ -20,7 +21,7 @@ const (
var (
schema = []string{
`create table if not exists kine
`CREATE TABLE IF NOT EXISTS kine
(
id INTEGER AUTO_INCREMENT,
name VARCHAR(630),
......@@ -33,14 +34,16 @@ var (
old_value MEDIUMBLOB,
PRIMARY KEY (id)
);`,
`CREATE INDEX kine_name_index ON kine (name)`,
`CREATE INDEX kine_name_id_index ON kine (name,id)`,
`CREATE INDEX kine_id_deleted_index ON kine (id,deleted)`,
`CREATE INDEX kine_prev_revision_index ON kine (prev_revision)`,
`CREATE UNIQUE INDEX kine_name_prev_revision_uindex ON kine (name, prev_revision)`,
}
nameIdx = "create index kine_name_index on kine (name)"
nameIDIdx = "create index kine_name_id_index on kine (name,id)"
revisionIdx = "create unique index kine_name_prev_revision_uindex on kine (name, prev_revision)"
createDB = "create database if not exists "
createDB = "CREATE DATABASE IF NOT EXISTS "
)
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
tlsConfig, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
......@@ -59,11 +62,30 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server
return nil, err
}
dialect, err := generic.Open(ctx, "mysql", parsedDSN, "?", false)
dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false)
if err != nil {
return nil, err
}
dialect.LastInsertID = true
dialect.CompactSQL = `
DELETE kv FROM kine AS kv
INNER JOIN (
SELECT kp.prev_revision AS id
FROM kine AS kp
WHERE
kp.prev_revision != 0 AND
kp.id <= ?
UNION
SELECT kd.id AS id
FROM kine AS kd
WHERE
kd.deleted != 0 AND
kd.id <= ?
) AS ks
ON
kv.id = ks.id AND
kv.name != 'compact_rev_key'`
dialect.TranslateErr = func(err error) error {
if err, ok := err.(*mysql.MySQLError); ok && err.Number == 1062 {
return server.ErrKeyExists
......@@ -79,24 +101,19 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server
}
func setup(db *sql.DB) error {
logrus.Infof("Configuring database table schema and indexes, this may take a moment...")
for _, stmt := range schema {
logrus.Tracef("SETUP EXEC : %v", generic.Stripped(stmt))
_, err := db.Exec(stmt)
if err != nil {
return err
if mysqlError, ok := err.(*mysql.MySQLError); !ok || mysqlError.Number != 1061 {
return err
}
}
}
// check if duplicate indexes
indexes := []string{
nameIdx,
nameIDIdx,
revisionIdx}
for _, idx := range indexes {
err := createIndex(db, idx)
if err != nil {
return err
}
}
logrus.Infof("Database tables and indexes are up to date")
return nil
}
......@@ -156,13 +173,3 @@ func prepareDSN(dataSourceName string, tlsConfig *cryptotls.Config) (string, err
return parsedDSN, nil
}
func createIndex(db *sql.DB, indexStmt string) error {
_, err := db.Exec(indexStmt)
if err != nil {
if mysqlError, ok := err.(*mysql.MySQLError); !ok || mysqlError.Number != 1061 {
return err
}
}
return nil
}
......@@ -14,6 +14,7 @@ import (
"github.com/rancher/kine/pkg/logstructured/sqllog"
"github.com/rancher/kine/pkg/server"
"github.com/rancher/kine/pkg/tls"
"github.com/sirupsen/logrus"
)
const (
......@@ -22,7 +23,7 @@ const (
var (
schema = []string{
`create table if not exists kine
`CREATE TABLE IF NOT EXISTS kine
(
id SERIAL PRIMARY KEY,
name VARCHAR(630),
......@@ -36,12 +37,14 @@ var (
);`,
`CREATE INDEX IF NOT EXISTS kine_name_index ON kine (name)`,
`CREATE INDEX IF NOT EXISTS kine_name_id_index ON kine (name,id)`,
`CREATE INDEX IF NOT EXISTS kine_id_deleted_index ON kine (id,deleted)`,
`CREATE INDEX IF NOT EXISTS kine_prev_revision_index ON kine (prev_revision)`,
`CREATE UNIQUE INDEX IF NOT EXISTS kine_name_prev_revision_uindex ON kine (name, prev_revision)`,
}
createDB = "create database "
createDB = "CREATE DATABASE "
)
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
parsedDSN, err := prepareDSN(dataSourceName, tlsInfo)
if err != nil {
return nil, err
......@@ -51,10 +54,28 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server
return nil, err
}
dialect, err := generic.Open(ctx, "postgres", parsedDSN, "$", true)
dialect, err := generic.Open(ctx, "postgres", parsedDSN, connPoolConfig, "$", true)
if err != nil {
return nil, err
}
dialect.CompactSQL = `
DELETE FROM kine AS kv
USING (
SELECT kp.prev_revision AS id
FROM kine AS kp
WHERE
kp.prev_revision != 0 AND
kp.id <= $1
UNION
SELECT kd.id AS id
FROM kine AS kd
WHERE
kd.deleted != 0 AND
kd.id <= $2
) AS ks
WHERE
kv.id = ks.id AND
kv.name != 'compact_rev_key'`
dialect.TranslateErr = func(err error) error {
if err, ok := err.(*pq.Error); ok && err.Code == "23505" {
return server.ErrKeyExists
......@@ -71,13 +92,17 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server
}
func setup(db *sql.DB) error {
logrus.Infof("Configuring database table schema and indexes, this may take a moment...")
for _, stmt := range schema {
logrus.Tracef("SETUP EXEC : %v", generic.Stripped(stmt))
_, err := db.Exec(stmt)
if err != nil {
return err
}
}
logrus.Infof("Database tables and indexes are up to date")
return nil
}
......@@ -110,7 +135,9 @@ func createDBIfNotExist(dataSourceName string) error {
return err
}
defer db.Close()
_, err = db.Exec(createDB + dbName + ";")
stmt := createDB + dbName + ";"
logrus.Tracef("SETUP EXEC : %v", generic.Stripped(stmt))
_, err = db.Exec(stmt)
if err != nil {
return err
}
......
......@@ -24,7 +24,7 @@ var (
schema = []string{
`CREATE TABLE IF NOT EXISTS kine
(
id INTEGER primary key autoincrement,
id INTEGER PRIMARY KEY AUTOINCREMENT,
name INTEGER,
created INTEGER,
deleted INTEGER,
......@@ -35,16 +35,19 @@ var (
old_value BLOB
)`,
`CREATE INDEX IF NOT EXISTS kine_name_index ON kine (name)`,
`CREATE INDEX IF NOT EXISTS kine_name_id_index ON kine (name,id)`,
`CREATE INDEX IF NOT EXISTS kine_id_deleted_index ON kine (id,deleted)`,
`CREATE INDEX IF NOT EXISTS kine_prev_revision_index ON kine (prev_revision)`,
`CREATE UNIQUE INDEX IF NOT EXISTS kine_name_prev_revision_uindex ON kine (name, prev_revision)`,
}
)
func New(ctx context.Context, dataSourceName string) (server.Backend, error) {
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName)
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig)
return backend, err
}
func NewVariant(ctx context.Context, driverName, dataSourceName string) (server.Backend, *generic.Generic, error) {
func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
if dataSourceName == "" {
if err := os.MkdirAll("./db", 0700); err != nil {
return nil, nil, err
......@@ -52,11 +55,29 @@ func NewVariant(ctx context.Context, driverName, dataSourceName string) (server.
dataSourceName = "./db/state.db?_journal=WAL&cache=shared"
}
dialect, err := generic.Open(ctx, driverName, dataSourceName, "?", false)
dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false)
if err != nil {
return nil, nil, err
}
dialect.LastInsertID = true
dialect.CompactSQL = `
DELETE FROM kine AS kv
WHERE
kv.name != 'compact_rev_key' AND
kv.id IN (
SELECT kp.prev_revision AS id
FROM kine AS kp
WHERE
kp.prev_revision != 0 AND
kp.id <= ?
UNION
SELECT kd.id AS id
FROM kine AS kd
WHERE
kd.deleted != 0 AND
kd.id <= ?
)`
dialect.TranslateErr = func(err error) error {
if err, ok := err.(sqlite3.Error); ok && err.ExtendedCode == sqlite3.ErrConstraintUnique {
return server.ErrKeyExists
......@@ -82,21 +103,22 @@ func NewVariant(ctx context.Context, driverName, dataSourceName string) (server.
if err != nil {
return nil, nil, errors.Wrap(err, "setup db")
}
//if err := setup(dialect.DB); err != nil {
// return nil, nil, errors.Wrap(err, "setup db")
//}
dialect.Migrate(context.Background())
return logstructured.New(sqllog.New(dialect)), dialect, nil
}
func setup(db *sql.DB) error {
logrus.Infof("Configuring database table schema and indexes, this may take a moment...")
for _, stmt := range schema {
logrus.Tracef("SETUP EXEC : %v", generic.Stripped(stmt))
_, err := db.Exec(stmt)
if err != nil {
return err
}
}
logrus.Infof("Database tables and indexes are up to date")
return nil
}
......@@ -13,11 +13,11 @@ import (
var errNoCgo = errors.New("this binary is built without CGO, sqlite is disabled")
func New(ctx context.Context, dataSourceName string) (server.Backend, error) {
func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
return nil, errNoCgo
}
func NewVariant(driverName, dataSourceName string) (server.Backend, *generic.Generic, error) {
func NewVariant(driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
return nil, nil, errNoCgo
}
......
......@@ -9,6 +9,7 @@ import (
"github.com/pkg/errors"
"github.com/rancher/kine/pkg/drivers/dqlite"
"github.com/rancher/kine/pkg/drivers/generic"
"github.com/rancher/kine/pkg/drivers/mysql"
"github.com/rancher/kine/pkg/drivers/pgsql"
"github.com/rancher/kine/pkg/drivers/sqlite"
......@@ -28,9 +29,10 @@ const (
)
type Config struct {
GRPCServer *grpc.Server
Listener string
Endpoint string
GRPCServer *grpc.Server
Listener string
Endpoint string
ConnectionPoolConfig generic.ConnectionPoolConfig
tls.Config
}
......@@ -124,13 +126,13 @@ func getKineStorageBackend(ctx context.Context, driver, dsn string, cfg Config)
switch driver {
case SQLiteBackend:
leaderElect = false
backend, err = sqlite.New(ctx, dsn)
backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
case DQLiteBackend:
backend, err = dqlite.New(ctx, dsn)
backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
case PostgresBackend:
backend, err = pgsql.New(ctx, dsn, cfg.Config)
backend, err = pgsql.New(ctx, dsn, cfg.Config, cfg.ConnectionPoolConfig)
case MySQLBackend:
backend, err = mysql.New(ctx, dsn, cfg.Config)
backend, err = mysql.New(ctx, dsn, cfg.Config, cfg.ConnectionPoolConfig)
default:
return false, nil, fmt.Errorf("storage backend is not defined")
}
......
......@@ -33,7 +33,12 @@ func (l *LogStructured) Start(ctx context.Context) error {
if err := l.log.Start(ctx); err != nil {
return err
}
l.Create(ctx, "/registry/health", []byte(`{"health":"true"}`), 0)
// See https://github.com/kubernetes/kubernetes/blob/442a69c3bdf6fe8e525b05887e57d89db1e2f3a5/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go#L97
if _, err := l.Create(ctx, "/registry/health", []byte(`{"health":"true"}`), 0); err != nil {
if err != server.ErrKeyExists {
logrus.Errorf("Failed to create health check key: %v", err)
}
}
go l.ttl(ctx)
return nil
}
......@@ -41,7 +46,7 @@ func (l *LogStructured) Start(ctx context.Context) error {
func (l *LogStructured) Get(ctx context.Context, key string, revision int64) (revRet int64, kvRet *server.KeyValue, errRet error) {
defer func() {
l.adjustRevision(ctx, &revRet)
logrus.Debugf("GET %s, rev=%d => rev=%d, kv=%v, err=%v", key, revision, revRet, kvRet != nil, errRet)
logrus.Tracef("GET %s, rev=%d => rev=%d, kv=%v, err=%v", key, revision, revRet, kvRet != nil, errRet)
}()
rev, event, err := l.get(ctx, key, revision, false)
......@@ -82,7 +87,7 @@ func (l *LogStructured) adjustRevision(ctx context.Context, rev *int64) {
func (l *LogStructured) Create(ctx context.Context, key string, value []byte, lease int64) (revRet int64, errRet error) {
defer func() {
l.adjustRevision(ctx, &revRet)
logrus.Debugf("CREATE %s, size=%d, lease=%d => rev=%d, err=%v", key, len(value), lease, revRet, errRet)
logrus.Tracef("CREATE %s, size=%d, lease=%d => rev=%d, err=%v", key, len(value), lease, revRet, errRet)
}()
rev, prevEvent, err := l.get(ctx, key, 0, true)
......@@ -114,7 +119,7 @@ func (l *LogStructured) Create(ctx context.Context, key string, value []byte, le
func (l *LogStructured) Delete(ctx context.Context, key string, revision int64) (revRet int64, kvRet *server.KeyValue, deletedRet bool, errRet error) {
defer func() {
l.adjustRevision(ctx, &revRet)
logrus.Debugf("DELETE %s, rev=%d => rev=%d, kv=%v, deleted=%v, err=%v", key, revision, revRet, kvRet != nil, deletedRet, errRet)
logrus.Tracef("DELETE %s, rev=%d => rev=%d, kv=%v, deleted=%v, err=%v", key, revision, revRet, kvRet != nil, deletedRet, errRet)
}()
rev, event, err := l.get(ctx, key, 0, true)
......@@ -155,7 +160,7 @@ func (l *LogStructured) Delete(ctx context.Context, key string, revision int64)
func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit, revision int64) (revRet int64, kvRet []*server.KeyValue, errRet error) {
defer func() {
logrus.Debugf("LIST %s, start=%s, limit=%d, rev=%d => rev=%d, kvs=%d, err=%v", prefix, startKey, limit, revision, revRet, len(kvRet), errRet)
logrus.Tracef("LIST %s, start=%s, limit=%d, rev=%d => rev=%d, kvs=%d, err=%v", prefix, startKey, limit, revision, revRet, len(kvRet), errRet)
}()
rev, events, err := l.log.List(ctx, prefix, startKey, limit, revision, false)
......@@ -185,7 +190,7 @@ func (l *LogStructured) List(ctx context.Context, prefix, startKey string, limit
func (l *LogStructured) Count(ctx context.Context, prefix string) (revRet int64, count int64, err error) {
defer func() {
logrus.Debugf("COUNT %s => rev=%d, count=%d, err=%v", prefix, revRet, count, err)
logrus.Tracef("COUNT %s => rev=%d, count=%d, err=%v", prefix, revRet, count, err)
}()
rev, count, err := l.log.Count(ctx, prefix)
if err != nil {
......@@ -211,7 +216,7 @@ func (l *LogStructured) Update(ctx context.Context, key string, value []byte, re
if kvRet != nil {
kvRev = kvRet.ModRevision
}
logrus.Debugf("UPDATE %s, value=%d, rev=%d, lease=%v => rev=%d, kvrev=%d, updated=%v, err=%v", key, len(value), revision, lease, revRet, kvRev, updateRet, errRet)
logrus.Tracef("UPDATE %s, value=%d, rev=%d, lease=%v => rev=%d, kvrev=%d, updated=%v, err=%v", key, len(value), revision, lease, revRet, kvRev, updateRet, errRet)
}()
rev, event, err := l.get(ctx, key, 0, false)
......@@ -304,14 +309,16 @@ func (l *LogStructured) ttl(ctx context.Context) {
case <-time.After(time.Duration(event.KV.Lease) * time.Second):
}
mutex.Lock()
l.Delete(ctx, event.KV.Key, event.KV.ModRevision)
if _, _, _, err := l.Delete(ctx, event.KV.Key, event.KV.ModRevision); err != nil {
logrus.Errorf("failed to delete expired key: %v", err)
}
mutex.Unlock()
}(event)
}
}
func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64) <-chan []*server.Event {
logrus.Debugf("WATCH %s, revision=%d", prefix, revision)
logrus.Tracef("WATCH %s, revision=%d", prefix, revision)
// starting watching right away so we don't miss anything
ctx, cancel := context.WithCancel(ctx)
......@@ -319,7 +326,7 @@ func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64
// include the current revision in list
if revision > 0 {
revision -= 1
revision--
}
result := make(chan []*server.Event, 100)
......@@ -330,7 +337,7 @@ func (l *LogStructured) Watch(ctx context.Context, prefix string, revision int64
cancel()
}
logrus.Debugf("WATCH LIST key=%s rev=%d => rev=%d kvs=%d", prefix, revision, rev, len(kvs))
logrus.Tracef("WATCH LIST key=%s rev=%d => rev=%d kvs=%d", prefix, revision, rev, len(kvs))
go func() {
lastRevision := revision
......
......@@ -4,9 +4,11 @@ import (
"context"
"go.etcd.io/etcd/etcdserver/etcdserverpb"
"go.etcd.io/etcd/mvcc/mvccpb"
)
func isCompact(txn *etcdserverpb.TxnRequest) bool {
// See https://github.com/kubernetes/kubernetes/blob/442a69c3bdf6fe8e525b05887e57d89db1e2f3a5/staging/src/k8s.io/apiserver/pkg/storage/etcd3/compact.go#L72
return len(txn.Compare) == 1 &&
txn.Compare[0].Target == etcdserverpb.Compare_VERSION &&
txn.Compare[0].Result == etcdserverpb.Compare_EQUAL &&
......@@ -18,14 +20,19 @@ func isCompact(txn *etcdserverpb.TxnRequest) bool {
}
func (l *LimitedServer) compact(ctx context.Context) (*etcdserverpb.TxnResponse, error) {
// return comparison failure so that the apiserver does not bother compacting
return &etcdserverpb.TxnResponse{
Header: &etcdserverpb.ResponseHeader{},
Succeeded: true,
Succeeded: false,
Responses: []*etcdserverpb.ResponseOp{
{
Response: &etcdserverpb.ResponseOp_ResponsePut{
ResponsePut: &etcdserverpb.PutResponse{
Response: &etcdserverpb.ResponseOp_ResponseRange{
ResponseRange: &etcdserverpb.RangeResponse{
Header: &etcdserverpb.ResponseHeader{},
Kvs: []*mvccpb.KeyValue{
&mvccpb.KeyValue{},
},
Count: 1,
},
},
},
......
......@@ -31,7 +31,7 @@ func (s *KVServerBridge) Watch(ws etcdserverpb.Watch_WatchServer) error {
if msg.GetCreateRequest() != nil {
w.Start(ws.Context(), msg.GetCreateRequest())
} else if msg.GetCancelRequest() != nil {
logrus.Debugf("WATCH CANCEL REQ id=%d", msg.GetCancelRequest().GetWatchId())
logrus.Tracef("WATCH CANCEL REQ id=%d", msg.GetCancelRequest().GetWatchId())
w.Cancel(msg.GetCancelRequest().WatchId, nil)
}
}
......@@ -58,7 +58,7 @@ func (w *watcher) Start(ctx context.Context, r *etcdserverpb.WatchCreateRequest)
key := string(r.Key)
logrus.Debugf("WATCH START id=%d, count=%d, key=%s, revision=%d", id, len(w.watches), key, r.StartRevision)
logrus.Tracef("WATCH START id=%d, count=%d, key=%s, revision=%d", id, len(w.watches), key, r.StartRevision)
go func() {
defer w.wg.Done()
......@@ -78,7 +78,7 @@ func (w *watcher) Start(ctx context.Context, r *etcdserverpb.WatchCreateRequest)
if logrus.IsLevelEnabled(logrus.DebugLevel) {
for _, event := range events {
logrus.Debugf("WATCH READ id=%d, key=%s, revision=%d", id, event.KV.Key, event.KV.ModRevision)
logrus.Tracef("WATCH READ id=%d, key=%s, revision=%d", id, event.KV.Key, event.KV.ModRevision)
}
}
......@@ -92,7 +92,7 @@ func (w *watcher) Start(ctx context.Context, r *etcdserverpb.WatchCreateRequest)
}
}
w.Cancel(id, nil)
logrus.Debugf("WATCH CLOSE id=%d, key=%s", id, key)
logrus.Tracef("WATCH CLOSE id=%d, key=%s", id, key)
}()
}
......@@ -130,7 +130,7 @@ func (w *watcher) Cancel(watchID int64, err error) {
if err != nil {
reason = err.Error()
}
logrus.Debugf("WATCH CANCEL id=%d reason=%s", watchID, reason)
logrus.Tracef("WATCH CANCEL id=%d reason=%s", watchID, reason)
serr := w.server.Send(&etcdserverpb.WatchResponse{
Header: &etcdserverpb.ResponseHeader{},
Canceled: true,
......
......@@ -603,7 +603,7 @@ github.com/konsorten/go-windows-terminal-sequences
# github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 => github.com/rancher/cri-tools v1.17.0-k3s1
github.com/kubernetes-sigs/cri-tools/cmd/crictl
github.com/kubernetes-sigs/cri-tools/pkg/version
# github.com/lib/pq v1.1.1
# github.com/lib/pq v1.8.0
github.com/lib/pq
github.com/lib/pq/oid
github.com/lib/pq/scram
......@@ -625,7 +625,7 @@ github.com/mailru/easyjson/jlexer
github.com/mailru/easyjson/jwriter
# github.com/mattn/go-shellwords v1.0.5
github.com/mattn/go-shellwords
# github.com/mattn/go-sqlite3 v1.13.0
# github.com/mattn/go-sqlite3 v1.14.4
github.com/mattn/go-sqlite3
# github.com/matttproud/golang_protobuf_extensions v1.0.1
github.com/matttproud/golang_protobuf_extensions/pbutil
......@@ -737,7 +737,7 @@ github.com/rancher/helm-controller/pkg/generated/informers/externalversions/helm
github.com/rancher/helm-controller/pkg/generated/informers/externalversions/internalinterfaces
github.com/rancher/helm-controller/pkg/generated/listers/helm.cattle.io/v1
github.com/rancher/helm-controller/pkg/helm
# github.com/rancher/kine v0.4.0
# github.com/rancher/kine v0.5.0
github.com/rancher/kine/pkg/broadcaster
github.com/rancher/kine/pkg/client
github.com/rancher/kine/pkg/drivers/dqlite
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment