diff --git a/integration/restart_test.go b/integration/restart_test.go index b3bcebc25..171c51bba 100644 --- a/integration/restart_test.go +++ b/integration/restart_test.go @@ -17,13 +17,11 @@ limitations under the License. package integration import ( - "os" - "os/exec" "sort" "testing" - "time" "github.com/containerd/containerd" + "github.com/containerd/containerd/errdefs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" @@ -127,7 +125,9 @@ func TestContainerdRestart(t *testing.T) { task, err := cntr.Task(ctx, nil) require.NoError(t, err) _, err = task.Delete(ctx, containerd.WithProcessKill) - require.NoError(t, err) + if err != nil { + require.True(t, errdefs.IsNotFound(err)) + } } } @@ -196,149 +196,4 @@ func TestContainerdRestart(t *testing.T) { } } -// Note: The test moves runc binary. -// The test requires: -// 1) The runtime is runc; -// 2) runc is in PATH; -func TestUnknownStateAfterContainerdRestart(t *testing.T) { - if *runtimeHandler != "" { - t.Skip("unsupported config: runtime handler is set") - } - runcPath, err := exec.LookPath("runc") - if err != nil { - t.Skip("unsupported config: runc not in PATH") - } - - sbConfig := PodSandboxConfig("sandbox", "sandbox-unknown-state") - - const testImage = "busybox" - t.Logf("Pull test image %q", testImage) - img, err := imageService.PullImage(&runtime.ImageSpec{Image: testImage}, nil, sbConfig) - require.NoError(t, err) - defer func() { - assert.NoError(t, imageService.RemoveImage(&runtime.ImageSpec{Image: img})) - }() - - t.Log("Should not be able to create sandbox without runc") - tmpRuncPath := Randomize(runcPath) - require.NoError(t, os.Rename(runcPath, tmpRuncPath)) - defer func() { - os.Rename(tmpRuncPath, runcPath) - }() - sb, err := runtimeService.RunPodSandbox(sbConfig, "") - if err == nil { - assert.NoError(t, runtimeService.StopPodSandbox(sb)) - assert.NoError(t, runtimeService.RemovePodSandbox(sb)) - t.Skip("unsupported config: runc is not being used") - } - require.NoError(t, os.Rename(tmpRuncPath, runcPath)) - - t.Log("Create a sandbox") - sb, err = runtimeService.RunPodSandbox(sbConfig, "") - require.NoError(t, err) - defer func() { - // Make sure the sandbox is cleaned up in any case. - runtimeService.StopPodSandbox(sb) - runtimeService.RemovePodSandbox(sb) - }() - ps, err := runtimeService.PodSandboxStatus(sb) - require.NoError(t, err) - assert.Equal(t, runtime.PodSandboxState_SANDBOX_READY, ps.GetState()) - - t.Log("Create a container") - cnConfig := ContainerConfig( - "container-unknown-state", - testImage, - WithCommand("sleep", "1000"), - ) - cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig) - require.NoError(t, err) - - t.Log("Start the container") - require.NoError(t, runtimeService.StartContainer(cn)) - cs, err := runtimeService.ContainerStatus(cn) - require.NoError(t, err) - assert.Equal(t, runtime.ContainerState_CONTAINER_RUNNING, cs.GetState()) - - t.Log("Move runc binary, so that container/sandbox can't be loaded after restart") - tmpRuncPath = Randomize(runcPath) - require.NoError(t, os.Rename(runcPath, tmpRuncPath)) - defer func() { - os.Rename(tmpRuncPath, runcPath) - }() - - t.Log("Restart containerd") - RestartContainerd(t) - - t.Log("Sandbox should be in NOTREADY state after containerd restart") - ps, err = runtimeService.PodSandboxStatus(sb) - require.NoError(t, err) - assert.Equal(t, runtime.PodSandboxState_SANDBOX_NOTREADY, ps.GetState()) - - t.Log("Container should be in UNKNOWN state after containerd restart") - cs, err = runtimeService.ContainerStatus(cn) - require.NoError(t, err) - assert.Equal(t, runtime.ContainerState_CONTAINER_UNKNOWN, cs.GetState()) - - t.Log("Stop/remove the sandbox should fail for the lack of runc") - assert.Error(t, runtimeService.StopPodSandbox(sb)) - assert.Error(t, runtimeService.RemovePodSandbox(sb)) - - t.Log("Stop/remove the container should fail for the lack of runc") - assert.Error(t, runtimeService.StopContainer(cn, 10)) - assert.Error(t, runtimeService.RemoveContainer(cn)) - - t.Log("Move runc back") - require.NoError(t, os.Rename(tmpRuncPath, runcPath)) - - t.Log("Sandbox should still be in NOTREADY state") - ps, err = runtimeService.PodSandboxStatus(sb) - require.NoError(t, err) - assert.Equal(t, runtime.PodSandboxState_SANDBOX_NOTREADY, ps.GetState()) - - t.Log("Container should still be in UNKNOWN state") - cs, err = runtimeService.ContainerStatus(cn) - require.NoError(t, err) - assert.Equal(t, runtime.ContainerState_CONTAINER_UNKNOWN, cs.GetState()) - - t.Log("Sandbox operations which require running state should fail") - _, err = runtimeService.PortForward(&runtime.PortForwardRequest{ - PodSandboxId: sb, - Port: []int32{8080}, - }) - assert.Error(t, err) - - t.Log("Container operations which require running state should fail") - assert.Error(t, runtimeService.ReopenContainerLog(cn)) - _, _, err = runtimeService.ExecSync(cn, []string{"ls"}, 10*time.Second) - assert.Error(t, err) - _, err = runtimeService.Attach(&runtime.AttachRequest{ - ContainerId: cn, - Stdin: true, - Stdout: true, - Stderr: true, - }) - assert.Error(t, err) - - t.Log("Containerd should still be running now") - _, err = runtimeService.Status() - require.NoError(t, err) - - t.Log("Remove the container should fail in this state") - assert.Error(t, runtimeService.RemoveContainer(cn)) - - t.Log("Remove the sandbox should fail in this state") - assert.Error(t, runtimeService.RemovePodSandbox(sb)) - - t.Log("Should be able to stop container in this state") - assert.NoError(t, runtimeService.StopContainer(cn, 10)) - - t.Log("Should be able to stop sandbox in this state") - assert.NoError(t, runtimeService.StopPodSandbox(sb)) - - t.Log("Should be able to remove container after stop") - assert.NoError(t, runtimeService.RemoveContainer(cn)) - - t.Log("Should be able to remove sandbox after stop") - assert.NoError(t, runtimeService.RemovePodSandbox(sb)) -} +// TODO: Add back the unknown state test. diff --git a/vendor.conf b/vendor.conf index 6da9ce2d0..271cdb10a 100644 --- a/vendor.conf +++ b/vendor.conf @@ -5,49 +5,54 @@ github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/distribution 0d3efadf0154c2b8a4e7b6621fff9809655cc580 # containerd dependencies -go.etcd.io/bbolt 2eb7227adea1d5cf85f0bc2a82b7059b13c2fa68 -google.golang.org/grpc 25c4f928eaa6d96443009bd842389fb4fa48664e # v1.20.1 -google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 -golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 -golang.org/x/sys 4c4f7f33c9ed00de01c4c741d2177abfcfe19307 https://github.com/golang/sys -golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e -golang.org/x/net f3200d17e092c607f615320ecaad13d87ad9a2b3 -github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c -github.com/syndtr/gocapability d98352740cb2c55f81556b63d4a1ec64c5a319c2 -github.com/sirupsen/logrus v1.4.1 -github.com/prometheus/procfs cb4147076ac75738c9a7d279075a253c0cc5acbd -github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563 -github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c -github.com/prometheus/client_golang f4fb1b73fb099f396a7f0036bf86aa8def4ed823 -github.com/pkg/errors v0.8.1 -github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 # v1.0.1-59-g29686db -github.com/opencontainers/runc f4982d86f7fde0b6f953cc62ccc4022c519a10a9 # v1.0.0-rc8-32-gf4982d86 -github.com/opencontainers/image-spec v1.0.1 -github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 -github.com/matttproud/golang_protobuf_extensions v1.0.1 -github.com/grpc-ecosystem/go-grpc-prometheus v1.1 -github.com/google/uuid v1.1.1 -github.com/golang/protobuf v1.2.0 -github.com/gogo/protobuf v1.2.1 -github.com/gogo/googleapis v1.2.0 -github.com/godbus/dbus v3 -github.com/docker/go-units v0.4.0 -github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098 -github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 -github.com/coreos/go-systemd v14 -github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 -github.com/containerd/ttrpc 92c8520ef9f86600c650dd540266a007bf03670f -github.com/containerd/go-runc e029b79d8cda8374981c64eba71f28ec38e5526f -github.com/containerd/fifo bda0ff6ed73c67bfb5e62bc9c697f146b7fd7f13 -github.com/containerd/continuity f2a389ac0a02ce21c09edd7344677a601970f41c -github.com/containerd/containerd d4802a64f9737f02db3426751f380d97fc878dec -github.com/containerd/console 0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f -github.com/containerd/cgroups c4b9ac5c7601384c965b9646fc515884e091ebb9 -github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 -github.com/Microsoft/hcsshim 9e921883ac929bbe515b39793ece99ce3a9d7706 -github.com/Microsoft/go-winio v0.4.14 github.com/BurntSushi/toml v0.3.1 -github.com/imdario/mergo v0.3.7 +github.com/Microsoft/go-winio v0.4.14 +github.com/Microsoft/hcsshim 9e921883ac929bbe515b39793ece99ce3a9d7706 +github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 +github.com/containerd/cgroups c4b9ac5c7601384c965b9646fc515884e091ebb9 +github.com/containerd/console 0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f +github.com/containerd/containerd v1.3.2 +github.com/containerd/continuity f2a389ac0a02ce21c09edd7344677a601970f41c +github.com/containerd/fifo bda0ff6ed73c67bfb5e62bc9c697f146b7fd7f13 +github.com/containerd/go-runc e029b79d8cda8374981c64eba71f28ec38e5526f +github.com/containerd/ttrpc 92c8520ef9f86600c650dd540266a007bf03670f +github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 +github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6 # v14 +github.com/cpuguy83/go-md2man 7762f7e404f8416dfa1d9bb6a8c192aa9acb4d19 # v1.0.10 +github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 +github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098 +github.com/docker/go-units v0.4.0 +github.com/godbus/dbus c7fdd8b5cd55e87b4e1f4e372cdb1db61dd6c66f # v3 +github.com/gogo/googleapis v1.2.0 +github.com/gogo/protobuf v1.2.1 +github.com/golang/protobuf v1.2.0 +github.com/google/uuid 0cd6bf5da1e1c83f8b45653022c74f71af0538a4 # v1.1.1 +github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 # v1.1 +github.com/hashicorp/golang-lru v0.5.3 +github.com/imdario/mergo 7c29201646fa3de8506f701213473dd407f19646 # v0.3.7 +github.com/matttproud/golang_protobuf_extensions v1.0.1 +github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 +github.com/opencontainers/image-spec v1.0.1 +github.com/opencontainers/runc d736ef14f0288d6993a1845745d6756cfc9ddd5a # v1.0.0-rc9 +github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 # v1.0.1-59-g29686db +github.com/pkg/errors v0.8.1 +github.com/prometheus/client_golang f4fb1b73fb099f396a7f0036bf86aa8def4ed823 +github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c +github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563 +github.com/prometheus/procfs cb4147076ac75738c9a7d279075a253c0cc5acbd +github.com/russross/blackfriday 05f3235734ad95d0016f6a23902f06461fcf567a # v1.5.2 +github.com/sirupsen/logrus v1.4.1 +github.com/syndtr/gocapability d98352740cb2c55f81556b63d4a1ec64c5a319c2 +github.com/urfave/cli v1.22.0 +go.etcd.io/bbolt v1.3.3 +go.opencensus.io v0.22.0 +golang.org/x/net f3200d17e092c607f615320ecaad13d87ad9a2b3 +golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e +golang.org/x/sys 9eafafc0a87e0fd0aeeba439a4573537970c44c7 https://github.com/golang/sys +golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 +google.golang.org/appengine v1.5.0 +google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 +google.golang.org/grpc 6eaf6f47437a6b4e2153a190160ef39a92c7eceb # v1.23.0 # kubernetes dependencies sigs.k8s.io/yaml v1.1.0 diff --git a/vendor/github.com/containerd/containerd/cmd/containerd/command/service_unsupported.go b/vendor/github.com/containerd/containerd/cmd/containerd/command/service_unsupported.go index 83c333b05..b28205255 100644 --- a/vendor/github.com/containerd/containerd/cmd/containerd/command/service_unsupported.go +++ b/vendor/github.com/containerd/containerd/cmd/containerd/command/service_unsupported.go @@ -29,7 +29,7 @@ func serviceFlags() []cli.Flag { return nil } -// applyPlatformFlags applys platform-specific flags. +// applyPlatformFlags applies platform-specific flags. func applyPlatformFlags(context *cli.Context) { } diff --git a/vendor/github.com/containerd/containerd/cmd/containerd/command/service_windows.go b/vendor/github.com/containerd/containerd/cmd/containerd/command/service_windows.go index f759a7131..697345910 100644 --- a/vendor/github.com/containerd/containerd/cmd/containerd/command/service_windows.go +++ b/vendor/github.com/containerd/containerd/cmd/containerd/command/service_windows.go @@ -42,6 +42,7 @@ var ( registerServiceFlag bool unregisterServiceFlag bool runServiceFlag bool + logFileFlag string kernel32 = windows.NewLazySystemDLL("kernel32.dll") setStdHandle = kernel32.NewProc("SetStdHandle") @@ -52,6 +53,8 @@ var ( service *handler ) +const defaultServiceName = "containerd" + // serviceFlags returns an array of flags for configuring containerd to run // as a Windows service under control of SCM. func serviceFlags() []cli.Flag { @@ -59,7 +62,7 @@ func serviceFlags() []cli.Flag { cli.StringFlag{ Name: "service-name", Usage: "Set the Windows service name", - Value: "containerd", + Value: defaultServiceName, }, cli.BoolFlag{ Name: "register-service", @@ -74,14 +77,18 @@ func serviceFlags() []cli.Flag { Usage: "", Hidden: true, }, + cli.StringFlag{ + Name: "log-file", + Usage: "Path to the containerd log file", + }, } } // applyPlatformFlags applies platform-specific flags. func applyPlatformFlags(context *cli.Context) { - - if s := context.GlobalString("service-name"); s != "" { - serviceNameFlag = s + serviceNameFlag = context.GlobalString("service-name") + if serviceNameFlag == "" { + serviceNameFlag = defaultServiceName } for _, v := range []struct { name string @@ -102,6 +109,7 @@ func applyPlatformFlags(context *cli.Context) { } { *v.d = context.GlobalBool(v.name) } + logFileFlag = context.GlobalString("log-file") } type handler struct { @@ -243,7 +251,15 @@ func registerUnregisterService(root string) (bool, error) { return true, err } - logrus.SetOutput(ioutil.Discard) + logOutput := ioutil.Discard + if logFileFlag != "" { + f, err := os.OpenFile(logFileFlag, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return true, errors.Wrapf(err, "open log file %q", logFileFlag) + } + logOutput = f + } + logrus.SetOutput(logOutput) } return false, nil } diff --git a/vendor/github.com/containerd/containerd/diff/apply/apply_linux.go b/vendor/github.com/containerd/containerd/diff/apply/apply_linux.go index c36b6090c..bbe9c17d9 100644 --- a/vendor/github.com/containerd/containerd/diff/apply/apply_linux.go +++ b/vendor/github.com/containerd/containerd/diff/apply/apply_linux.go @@ -26,12 +26,18 @@ import ( "github.com/containerd/containerd/archive" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/mount" + "github.com/opencontainers/runc/libcontainer/system" "github.com/pkg/errors" ) func apply(ctx context.Context, mounts []mount.Mount, r io.Reader) error { switch { case len(mounts) == 1 && mounts[0].Type == "overlay": + // OverlayConvertWhiteout (mknod c 0 0) doesn't work in userns. + // https://github.com/containerd/containerd/issues/3762 + if system.RunningInUserNS() { + break + } path, parents, err := getOverlayPath(mounts[0].Options) if err != nil { if errdefs.IsInvalidArgument(err) { diff --git a/vendor/github.com/containerd/containerd/diff/stream.go b/vendor/github.com/containerd/containerd/diff/stream.go index 4b8f27f14..1b625feaa 100644 --- a/vendor/github.com/containerd/containerd/diff/stream.go +++ b/vendor/github.com/containerd/containerd/diff/stream.go @@ -88,11 +88,11 @@ type StreamProcessor interface { } func compressedHandler(ctx context.Context, mediaType string) (StreamProcessorInit, bool) { - compressed, err := images.IsCompressedDiff(ctx, mediaType) + compressed, err := images.DiffCompression(ctx, mediaType) if err != nil { return nil, false } - if compressed { + if compressed != "" { return func(ctx context.Context, stream StreamProcessor, payloads map[string]*types.Any) (StreamProcessor, error) { ds, err := compression.DecompressStream(stream) if err != nil { diff --git a/vendor/github.com/containerd/containerd/images/archive/importer.go b/vendor/github.com/containerd/containerd/images/archive/importer.go index 539900685..5bc887130 100644 --- a/vendor/github.com/containerd/containerd/images/archive/importer.go +++ b/vendor/github.com/containerd/containerd/images/archive/importer.go @@ -124,7 +124,7 @@ func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opt } // If OCI layout was given, interpret the tar as an OCI layout. - // When not provided, the layout of the tar will be interpretted + // When not provided, the layout of the tar will be interpreted // as Docker v1.1 or v1.2. if ociLayout.Version != "" { if ociLayout.Version != ocispec.ImageLayoutVersion { diff --git a/vendor/github.com/containerd/containerd/images/image.go b/vendor/github.com/containerd/containerd/images/image.go index 7d4f39c0a..ee5778d24 100644 --- a/vendor/github.com/containerd/containerd/images/image.go +++ b/vendor/github.com/containerd/containerd/images/image.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "sort" - "strings" "time" "github.com/containerd/containerd/content" @@ -358,16 +357,11 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr } descs = append(descs, index.Manifests...) - case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip, - MediaTypeDockerSchema2LayerEnc, MediaTypeDockerSchema2LayerGzipEnc, - MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip, - MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig, - ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, - MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig: - // childless data types. - return nil, nil default: + if IsLayerType(desc.MediaType) || IsKnownConfig(desc.MediaType) { + // childless data types. + return nil, nil + } log.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType) } @@ -390,72 +384,3 @@ func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.D } return config.RootFS.DiffIDs, nil } - -// IsCompressedDiff returns true if mediaType is a known compressed diff media type. -// It returns false if the media type is a diff, but not compressed. If the media type -// is not a known diff type, it returns errdefs.ErrNotImplemented -func IsCompressedDiff(ctx context.Context, mediaType string) (bool, error) { - switch mediaType { - case ocispec.MediaTypeImageLayer, MediaTypeDockerSchema2Layer: - case ocispec.MediaTypeImageLayerGzip, MediaTypeDockerSchema2LayerGzip: - return true, nil - default: - // Still apply all generic media types *.tar[.+]gzip and *.tar - if strings.HasSuffix(mediaType, ".tar.gzip") || strings.HasSuffix(mediaType, ".tar+gzip") { - return true, nil - } else if !strings.HasSuffix(mediaType, ".tar") { - return false, errdefs.ErrNotImplemented - } - } - return false, nil -} - -// GetImageLayerDescriptors gets the image layer Descriptors of an image; the array contains -// a list of Descriptors belonging to one platform followed by lists of other platforms -func GetImageLayerDescriptors(ctx context.Context, cs content.Store, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - var lis []ocispec.Descriptor - - ds := platforms.DefaultSpec() - platform := &ds - - switch desc.MediaType { - case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex, - MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - children, err := Children(ctx, cs, desc) - if err != nil { - if errdefs.IsNotFound(err) { - return []ocispec.Descriptor{}, nil - } - return []ocispec.Descriptor{}, err - } - - if desc.Platform != nil { - platform = desc.Platform - } - - for _, child := range children { - var tmp []ocispec.Descriptor - - switch child.MediaType { - case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2Layer, - ocispec.MediaTypeImageLayerGzip, ocispec.MediaTypeImageLayer, - MediaTypeDockerSchema2LayerGzipEnc, MediaTypeDockerSchema2LayerEnc: - tdesc := child - tdesc.Platform = platform - tmp = append(tmp, tdesc) - default: - tmp, err = GetImageLayerDescriptors(ctx, cs, child) - } - - if err != nil { - return []ocispec.Descriptor{}, err - } - - lis = append(lis, tmp...) - } - case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: - default: - return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "GetImageLayerInfo: unhandled media type %s", desc.MediaType) - } - return lis, nil -} diff --git a/vendor/github.com/containerd/containerd/images/mediatypes.go b/vendor/github.com/containerd/containerd/images/mediatypes.go index 5fee7467a..2f47b0e68 100644 --- a/vendor/github.com/containerd/containerd/images/mediatypes.go +++ b/vendor/github.com/containerd/containerd/images/mediatypes.go @@ -16,16 +16,23 @@ package images +import ( + "context" + "sort" + "strings" + + "github.com/containerd/containerd/errdefs" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + // mediatype definitions for image components handled in containerd. // // oci components are generally referenced directly, although we may centralize // here for clarity. const ( MediaTypeDockerSchema2Layer = "application/vnd.docker.image.rootfs.diff.tar" - MediaTypeDockerSchema2LayerEnc = "application/vnd.docker.image.rootfs.diff.tar+enc" MediaTypeDockerSchema2LayerForeign = "application/vnd.docker.image.rootfs.foreign.diff.tar" MediaTypeDockerSchema2LayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip" - MediaTypeDockerSchema2LayerGzipEnc = "application/vnd.docker.image.rootfs.diff.tar.gzip+enc" MediaTypeDockerSchema2LayerForeignGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" MediaTypeDockerSchema2Config = "application/vnd.docker.container.image.v1+json" MediaTypeDockerSchema2Manifest = "application/vnd.docker.distribution.manifest.v2+json" @@ -42,3 +49,78 @@ const ( // Legacy Docker schema1 manifest MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" ) + +// DiffCompression returns the compression as defined by the layer diff media +// type. For Docker media types without compression, "unknown" is returned to +// indicate that the media type may be compressed. If the media type is not +// recognized as a layer diff, then it returns errdefs.ErrNotImplemented +func DiffCompression(ctx context.Context, mediaType string) (string, error) { + base, ext := parseMediaTypes(mediaType) + switch base { + case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerForeign: + if len(ext) > 0 { + // Type is wrapped + return "", nil + } + // These media types may have been compressed but failed to + // use the correct media type. The decompression function + // should detect and handle this case. + return "unknown", nil + case MediaTypeDockerSchema2LayerGzip, MediaTypeDockerSchema2LayerForeignGzip: + if len(ext) > 0 { + // Type is wrapped + return "", nil + } + return "gzip", nil + case ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerNonDistributable: + if len(ext) > 0 { + switch ext[len(ext)-1] { + case "gzip": + return "gzip", nil + } + } + return "", nil + default: + return "", errdefs.ErrNotImplemented + } +} + +// parseMediaTypes splits the media type into the base type and +// an array of sorted extensions +func parseMediaTypes(mt string) (string, []string) { + if mt == "" { + return "", []string{} + } + + s := strings.Split(mt, "+") + ext := s[1:] + sort.Strings(ext) + + return s[0], ext +} + +// IsLayerTypes returns true if the media type is a layer +func IsLayerType(mt string) bool { + if strings.HasPrefix(mt, "application/vnd.oci.image.layer.") { + return true + } + + // Parse Docker media types, strip off any + suffixes first + base, _ := parseMediaTypes(mt) + switch base { + case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip, + MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip: + return true + } + return false +} + +// IsKnownConfig returns true if the media type is a known config type +func IsKnownConfig(mt string) bool { + switch mt { + case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig, + MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig: + return true + } + return false +} diff --git a/vendor/github.com/containerd/containerd/install.go b/vendor/github.com/containerd/containerd/install.go index 5b8b735de..df6c8bc8a 100644 --- a/vendor/github.com/containerd/containerd/install.go +++ b/vendor/github.com/containerd/containerd/install.go @@ -21,6 +21,8 @@ import ( "context" "os" "path/filepath" + "runtime" + "strings" introspectionapi "github.com/containerd/containerd/api/services/introspection/v1" "github.com/containerd/containerd/archive" @@ -48,6 +50,15 @@ func (c *Client) Install(ctx context.Context, image Image, opts ...InstallOpts) if err != nil { return err } + + var binDir, libDir string + if runtime.GOOS == "windows" { + binDir = "Files\\bin" + libDir = "Files\\lib" + } else { + binDir = "bin" + libDir = "lib" + } for _, layer := range manifest.Layers { ra, err := cs.ReaderAt(ctx, layer) if err != nil { @@ -60,9 +71,14 @@ func (c *Client) Install(ctx context.Context, image Image, opts ...InstallOpts) } if _, err := archive.Apply(ctx, path, r, archive.WithFilter(func(hdr *tar.Header) (bool, error) { d := filepath.Dir(hdr.Name) - result := d == "bin" + result := d == binDir + if config.Libs { - result = result || d == "lib" + result = result || d == libDir + } + + if runtime.GOOS == "windows" { + hdr.Name = strings.Replace(hdr.Name, "Files", "", 1) } if result && !config.Replace { if _, err := os.Lstat(filepath.Join(path, hdr.Name)); err == nil { diff --git a/vendor/github.com/containerd/containerd/metadata/containers.go b/vendor/github.com/containerd/containerd/metadata/containers.go index af8224786..09b0d203d 100644 --- a/vendor/github.com/containerd/containerd/metadata/containers.go +++ b/vendor/github.com/containerd/containerd/metadata/containers.go @@ -19,6 +19,7 @@ package metadata import ( "context" "strings" + "sync/atomic" "time" "github.com/containerd/containerd/containers" @@ -35,13 +36,13 @@ import ( ) type containerStore struct { - tx *bolt.Tx + db *DB } // NewContainerStore returns a Store backed by an underlying bolt DB -func NewContainerStore(tx *bolt.Tx) containers.Store { +func NewContainerStore(db *DB) containers.Store { return &containerStore{ - tx: tx, + db: db, } } @@ -51,14 +52,21 @@ func (s *containerStore) Get(ctx context.Context, id string) (containers.Contain return containers.Container{}, err } - bkt := getContainerBucket(s.tx, namespace, id) - if bkt == nil { - return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "container %q in namespace %q", id, namespace) - } - container := containers.Container{ID: id} - if err := readContainer(&container, bkt); err != nil { - return containers.Container{}, errors.Wrapf(err, "failed to read container %q", id) + + if err := view(ctx, s.db, func(tx *bolt.Tx) error { + bkt := getContainerBucket(tx, namespace, id) + if bkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "container %q in namespace %q", id, namespace) + } + + if err := readContainer(&container, bkt); err != nil { + return errors.Wrapf(err, "failed to read container %q", id) + } + + return nil + }); err != nil { + return containers.Container{}, err } return container, nil @@ -75,27 +83,30 @@ func (s *containerStore) List(ctx context.Context, fs ...string) ([]containers.C return nil, errors.Wrap(errdefs.ErrInvalidArgument, err.Error()) } - bkt := getContainersBucket(s.tx, namespace) - if bkt == nil { - return nil, nil // empty store - } - var m []containers.Container - if err := bkt.ForEach(func(k, v []byte) error { - cbkt := bkt.Bucket(k) - if cbkt == nil { - return nil - } - container := containers.Container{ID: string(k)} - if err := readContainer(&container, cbkt); err != nil { - return errors.Wrapf(err, "failed to read container %q", string(k)) + if err := view(ctx, s.db, func(tx *bolt.Tx) error { + bkt := getContainersBucket(tx, namespace) + if bkt == nil { + return nil // empty store } - if filter.Match(adaptContainer(container)) { - m = append(m, container) - } - return nil + return bkt.ForEach(func(k, v []byte) error { + cbkt := bkt.Bucket(k) + if cbkt == nil { + return nil + } + container := containers.Container{ID: string(k)} + + if err := readContainer(&container, cbkt); err != nil { + return errors.Wrapf(err, "failed to read container %q", string(k)) + } + + if filter.Match(adaptContainer(container)) { + m = append(m, container) + } + return nil + }) }); err != nil { return nil, err } @@ -113,23 +124,29 @@ func (s *containerStore) Create(ctx context.Context, container containers.Contai return containers.Container{}, errors.Wrap(err, "create container failed validation") } - bkt, err := createContainersBucket(s.tx, namespace) - if err != nil { - return containers.Container{}, err - } + if err := update(ctx, s.db, func(tx *bolt.Tx) error { + bkt, err := createContainersBucket(tx, namespace) + if err != nil { + return err + } - cbkt, err := bkt.CreateBucket([]byte(container.ID)) - if err != nil { - if err == bolt.ErrBucketExists { - err = errors.Wrapf(errdefs.ErrAlreadyExists, "container %q", container.ID) + cbkt, err := bkt.CreateBucket([]byte(container.ID)) + if err != nil { + if err == bolt.ErrBucketExists { + err = errors.Wrapf(errdefs.ErrAlreadyExists, "container %q", container.ID) + } + return err } - return containers.Container{}, err - } - container.CreatedAt = time.Now().UTC() - container.UpdatedAt = container.CreatedAt - if err := writeContainer(cbkt, &container); err != nil { - return containers.Container{}, errors.Wrapf(err, "failed to write container %q", container.ID) + container.CreatedAt = time.Now().UTC() + container.UpdatedAt = container.CreatedAt + if err := writeContainer(cbkt, &container); err != nil { + return errors.Wrapf(err, "failed to write container %q", container.ID) + } + + return nil + }); err != nil { + return containers.Container{}, err } return container, nil @@ -145,85 +162,91 @@ func (s *containerStore) Update(ctx context.Context, container containers.Contai return containers.Container{}, errors.Wrapf(errdefs.ErrInvalidArgument, "must specify a container id") } - bkt := getContainersBucket(s.tx, namespace) - if bkt == nil { - return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "cannot update container %q in namespace %q", container.ID, namespace) - } - - cbkt := bkt.Bucket([]byte(container.ID)) - if cbkt == nil { - return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "container %q", container.ID) - } - var updated containers.Container - if err := readContainer(&updated, cbkt); err != nil { - return updated, errors.Wrapf(err, "failed to read container %q", container.ID) - } - createdat := updated.CreatedAt - updated.ID = container.ID - - if len(fieldpaths) == 0 { - // only allow updates to these field on full replace. - fieldpaths = []string{"labels", "spec", "extensions", "image", "snapshotkey"} - - // Fields that are immutable must cause an error when no field paths - // are provided. This allows these fields to become mutable in the - // future. - if updated.Snapshotter != container.Snapshotter { - return containers.Container{}, errors.Wrapf(errdefs.ErrInvalidArgument, "container.Snapshotter field is immutable") + if err := update(ctx, s.db, func(tx *bolt.Tx) error { + bkt := getContainersBucket(tx, namespace) + if bkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "cannot update container %q in namespace %q", container.ID, namespace) } - if updated.Runtime.Name != container.Runtime.Name { - return containers.Container{}, errors.Wrapf(errdefs.ErrInvalidArgument, "container.Runtime.Name field is immutable") + cbkt := bkt.Bucket([]byte(container.ID)) + if cbkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "container %q", container.ID) } - } - // apply the field mask. If you update this code, you better follow the - // field mask rules in field_mask.proto. If you don't know what this - // is, do not update this code. - for _, path := range fieldpaths { - if strings.HasPrefix(path, "labels.") { - if updated.Labels == nil { - updated.Labels = map[string]string{} + if err := readContainer(&updated, cbkt); err != nil { + return errors.Wrapf(err, "failed to read container %q", container.ID) + } + createdat := updated.CreatedAt + updated.ID = container.ID + + if len(fieldpaths) == 0 { + // only allow updates to these field on full replace. + fieldpaths = []string{"labels", "spec", "extensions", "image", "snapshotkey"} + + // Fields that are immutable must cause an error when no field paths + // are provided. This allows these fields to become mutable in the + // future. + if updated.Snapshotter != container.Snapshotter { + return errors.Wrapf(errdefs.ErrInvalidArgument, "container.Snapshotter field is immutable") + } + + if updated.Runtime.Name != container.Runtime.Name { + return errors.Wrapf(errdefs.ErrInvalidArgument, "container.Runtime.Name field is immutable") } - key := strings.TrimPrefix(path, "labels.") - updated.Labels[key] = container.Labels[key] - continue } - if strings.HasPrefix(path, "extensions.") { - if updated.Extensions == nil { - updated.Extensions = map[string]types.Any{} + // apply the field mask. If you update this code, you better follow the + // field mask rules in field_mask.proto. If you don't know what this + // is, do not update this code. + for _, path := range fieldpaths { + if strings.HasPrefix(path, "labels.") { + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + key := strings.TrimPrefix(path, "labels.") + updated.Labels[key] = container.Labels[key] + continue + } + + if strings.HasPrefix(path, "extensions.") { + if updated.Extensions == nil { + updated.Extensions = map[string]types.Any{} + } + key := strings.TrimPrefix(path, "extensions.") + updated.Extensions[key] = container.Extensions[key] + continue + } + + switch path { + case "labels": + updated.Labels = container.Labels + case "spec": + updated.Spec = container.Spec + case "extensions": + updated.Extensions = container.Extensions + case "image": + updated.Image = container.Image + case "snapshotkey": + updated.SnapshotKey = container.SnapshotKey + default: + return errors.Wrapf(errdefs.ErrInvalidArgument, "cannot update %q field on %q", path, container.ID) } - key := strings.TrimPrefix(path, "extensions.") - updated.Extensions[key] = container.Extensions[key] - continue } - switch path { - case "labels": - updated.Labels = container.Labels - case "spec": - updated.Spec = container.Spec - case "extensions": - updated.Extensions = container.Extensions - case "image": - updated.Image = container.Image - case "snapshotkey": - updated.SnapshotKey = container.SnapshotKey - default: - return containers.Container{}, errors.Wrapf(errdefs.ErrInvalidArgument, "cannot update %q field on %q", path, container.ID) + if err := validateContainer(&updated); err != nil { + return errors.Wrap(err, "update failed validation") } - } - if err := validateContainer(&updated); err != nil { - return containers.Container{}, errors.Wrap(err, "update failed validation") - } + updated.CreatedAt = createdat + updated.UpdatedAt = time.Now().UTC() + if err := writeContainer(cbkt, &updated); err != nil { + return errors.Wrapf(err, "failed to write container %q", container.ID) + } - updated.CreatedAt = createdat - updated.UpdatedAt = time.Now().UTC() - if err := writeContainer(cbkt, &updated); err != nil { - return containers.Container{}, errors.Wrapf(err, "failed to write container %q", container.ID) + return nil + }); err != nil { + return containers.Container{}, err } return updated, nil @@ -235,15 +258,23 @@ func (s *containerStore) Delete(ctx context.Context, id string) error { return err } - bkt := getContainersBucket(s.tx, namespace) - if bkt == nil { - return errors.Wrapf(errdefs.ErrNotFound, "cannot delete container %q in namespace %q", id, namespace) - } + return update(ctx, s.db, func(tx *bolt.Tx) error { + bkt := getContainersBucket(tx, namespace) + if bkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "cannot delete container %q in namespace %q", id, namespace) + } - if err := bkt.DeleteBucket([]byte(id)); err == bolt.ErrBucketNotFound { - return errors.Wrapf(errdefs.ErrNotFound, "container %v", id) - } - return err + if err := bkt.DeleteBucket([]byte(id)); err != nil { + if err == bolt.ErrBucketNotFound { + err = errors.Wrapf(errdefs.ErrNotFound, "container %v", id) + } + return err + } + + atomic.AddUint32(&s.db.dirty, 1) + + return nil + }) } func validateContainer(container *containers.Container) error { diff --git a/vendor/github.com/containerd/containerd/metadata/content.go b/vendor/github.com/containerd/containerd/metadata/content.go index 4a07a256b..268a9b1b7 100644 --- a/vendor/github.com/containerd/containerd/metadata/content.go +++ b/vendor/github.com/containerd/containerd/metadata/content.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "strings" "sync" + "sync/atomic" "time" "github.com/containerd/containerd/content" @@ -221,9 +222,8 @@ func (cs *contentStore) Delete(ctx context.Context, dgst digest.Digest) error { } // Mark content store as dirty for triggering garbage collection - cs.db.dirtyL.Lock() + atomic.AddUint32(&cs.db.dirty, 1) cs.db.dirtyCS = true - cs.db.dirtyL.Unlock() return nil }) diff --git a/vendor/github.com/containerd/containerd/metadata/db.go b/vendor/github.com/containerd/containerd/metadata/db.go index 7f1b27b38..40d045f05 100644 --- a/vendor/github.com/containerd/containerd/metadata/db.go +++ b/vendor/github.com/containerd/containerd/metadata/db.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "strings" "sync" + "sync/atomic" "time" "github.com/containerd/containerd/content" @@ -75,10 +76,16 @@ type DB struct { // sweep phases without preventing read transactions. wlock sync.RWMutex - // dirty flags and lock keeps track of datastores which have had deletions - // since the last garbage collection. These datastores will will be garbage - // collected during the next garbage collection. - dirtyL sync.Mutex + // dirty flag indicates that references have been removed which require + // a garbage collection to ensure the database is clean. This tracks + // the number of dirty operations. This should be updated and read + // atomically if outside of wlock.Lock. + dirty uint32 + + // dirtySS and dirtyCS flags keeps track of datastores which have had + // deletions since the last garbage collection. These datastores will + // be garbage collected during the next garbage collection. These + // should only be updated inside of a write transaction or wlock.Lock. dirtySS map[string]struct{} dirtyCS bool @@ -162,7 +169,7 @@ func (m *DB) Init(ctx context.Context) error { } } - // Previous version fo database found + // Previous version of database found if schema != "v0" { updates := migrations[i:] @@ -237,12 +244,10 @@ func (m *DB) Update(fn func(*bolt.Tx) error) error { defer m.wlock.RUnlock() err := m.db.Update(fn) if err == nil { - m.dirtyL.Lock() - dirty := m.dirtyCS || len(m.dirtySS) > 0 + dirty := atomic.LoadUint32(&m.dirty) > 0 for _, fn := range m.mutationCallbacks { fn(dirty) } - m.dirtyL.Unlock() } return err @@ -254,9 +259,9 @@ func (m *DB) Update(fn func(*bolt.Tx) error) error { // The callback function is an argument for whether a deletion has occurred // since the last garbage collection. func (m *DB) RegisterMutationCallback(fn func(bool)) { - m.dirtyL.Lock() + m.wlock.Lock() m.mutationCallbacks = append(m.mutationCallbacks, fn) - m.dirtyL.Unlock() + m.wlock.Unlock() } // GCStats holds the duration for the different phases of the garbage collector @@ -282,8 +287,6 @@ func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error) { return nil, err } - m.dirtyL.Lock() - if err := m.db.Update(func(tx *bolt.Tx) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -309,7 +312,6 @@ func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error) { return nil }); err != nil { - m.dirtyL.Unlock() m.wlock.Unlock() return nil, err } @@ -317,6 +319,9 @@ func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error) { var stats GCStats var wg sync.WaitGroup + // reset dirty, no need for atomic inside of wlock.Lock + m.dirty = 0 + if len(m.dirtySS) > 0 { var sl sync.Mutex stats.SnapshotD = map[string]time.Duration{} @@ -349,8 +354,6 @@ func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error) { m.dirtyCS = false } - m.dirtyL.Unlock() - stats.MetaD = time.Since(t1) m.wlock.Unlock() diff --git a/vendor/github.com/containerd/containerd/metadata/images.go b/vendor/github.com/containerd/containerd/metadata/images.go index 1dda753db..cace4e180 100644 --- a/vendor/github.com/containerd/containerd/metadata/images.go +++ b/vendor/github.com/containerd/containerd/metadata/images.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "fmt" "strings" + "sync/atomic" "time" "github.com/containerd/containerd/errdefs" @@ -249,19 +250,16 @@ func (s *imageStore) Delete(ctx context.Context, name string, opts ...images.Del return errors.Wrapf(errdefs.ErrNotFound, "image %q", name) } - err = bkt.DeleteBucket([]byte(name)) - if err == bolt.ErrBucketNotFound { - return errors.Wrapf(errdefs.ErrNotFound, "image %q", name) + if err = bkt.DeleteBucket([]byte(name)); err != nil { + if err == bolt.ErrBucketNotFound { + err = errors.Wrapf(errdefs.ErrNotFound, "image %q", name) + } + return err } - // A reference to a piece of content has been removed, - // mark content store as dirty for triggering garbage - // collection - s.db.dirtyL.Lock() - s.db.dirtyCS = true - s.db.dirtyL.Unlock() + atomic.AddUint32(&s.db.dirty, 1) - return err + return nil }) } diff --git a/vendor/github.com/containerd/containerd/metadata/leases.go b/vendor/github.com/containerd/containerd/metadata/leases.go index cd8809f4c..60da06b0f 100644 --- a/vendor/github.com/containerd/containerd/metadata/leases.go +++ b/vendor/github.com/containerd/containerd/metadata/leases.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync/atomic" "time" "github.com/containerd/containerd/errdefs" @@ -35,14 +36,14 @@ import ( // LeaseManager manages the create/delete lifecycle of leases // and also returns existing leases type LeaseManager struct { - tx *bolt.Tx + db *DB } // NewLeaseManager creates a new lease manager for managing leases using // the provided database transaction. -func NewLeaseManager(tx *bolt.Tx) *LeaseManager { +func NewLeaseManager(db *DB) *LeaseManager { return &LeaseManager{ - tx: tx, + db: db, } } @@ -63,35 +64,40 @@ func (lm *LeaseManager) Create(ctx context.Context, opts ...leases.Opt) (leases. return leases.Lease{}, err } - topbkt, err := createBucketIfNotExists(lm.tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases) - if err != nil { - return leases.Lease{}, err - } + if err := update(ctx, lm.db, func(tx *bolt.Tx) error { + topbkt, err := createBucketIfNotExists(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases) + if err != nil { + return err + } - txbkt, err := topbkt.CreateBucket([]byte(l.ID)) - if err != nil { - if err == bolt.ErrBucketExists { - err = errdefs.ErrAlreadyExists + txbkt, err := topbkt.CreateBucket([]byte(l.ID)) + if err != nil { + if err == bolt.ErrBucketExists { + err = errdefs.ErrAlreadyExists + } + return errors.Wrapf(err, "lease %q", l.ID) } - return leases.Lease{}, errors.Wrapf(err, "lease %q", l.ID) - } - t := time.Now().UTC() - createdAt, err := t.MarshalBinary() - if err != nil { - return leases.Lease{}, err - } - if err := txbkt.Put(bucketKeyCreatedAt, createdAt); err != nil { - return leases.Lease{}, err - } + t := time.Now().UTC() + createdAt, err := t.MarshalBinary() + if err != nil { + return err + } + if err := txbkt.Put(bucketKeyCreatedAt, createdAt); err != nil { + return err + } - if l.Labels != nil { - if err := boltutil.WriteLabels(txbkt, l.Labels); err != nil { - return leases.Lease{}, err + if l.Labels != nil { + if err := boltutil.WriteLabels(txbkt, l.Labels); err != nil { + return err + } } - } - l.CreatedAt = t + l.CreatedAt = t + return nil + }); err != nil { + return leases.Lease{}, err + } return l, nil } @@ -102,17 +108,22 @@ func (lm *LeaseManager) Delete(ctx context.Context, lease leases.Lease, _ ...lea return err } - topbkt := getBucket(lm.tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases) - if topbkt == nil { - return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) - } - if err := topbkt.DeleteBucket([]byte(lease.ID)); err != nil { - if err == bolt.ErrBucketNotFound { - err = errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) + return update(ctx, lm.db, func(tx *bolt.Tx) error { + topbkt := getBucket(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases) + if topbkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) } - return err - } - return nil + if err := topbkt.DeleteBucket([]byte(lease.ID)); err != nil { + if err == bolt.ErrBucketNotFound { + err = errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) + } + return err + } + + atomic.AddUint32(&lm.db.dirty, 1) + + return nil + }) } // List lists all active leases @@ -129,39 +140,41 @@ func (lm *LeaseManager) List(ctx context.Context, fs ...string) ([]leases.Lease, var ll []leases.Lease - topbkt := getBucket(lm.tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases) - if topbkt == nil { - return ll, nil - } - - if err := topbkt.ForEach(func(k, v []byte) error { - if v != nil { + if err := view(ctx, lm.db, func(tx *bolt.Tx) error { + topbkt := getBucket(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases) + if topbkt == nil { return nil } - txbkt := topbkt.Bucket(k) - l := leases.Lease{ - ID: string(k), - } + return topbkt.ForEach(func(k, v []byte) error { + if v != nil { + return nil + } + txbkt := topbkt.Bucket(k) - if v := txbkt.Get(bucketKeyCreatedAt); v != nil { - t := &l.CreatedAt - if err := t.UnmarshalBinary(v); err != nil { - return err + l := leases.Lease{ + ID: string(k), } - } - labels, err := boltutil.ReadLabels(txbkt) - if err != nil { - return err - } - l.Labels = labels + if v := txbkt.Get(bucketKeyCreatedAt); v != nil { + t := &l.CreatedAt + if err := t.UnmarshalBinary(v); err != nil { + return err + } + } - if filter.Match(adaptLease(l)) { - ll = append(ll, l) - } + labels, err := boltutil.ReadLabels(txbkt) + if err != nil { + return err + } + l.Labels = labels - return nil + if filter.Match(adaptLease(l)) { + ll = append(ll, l) + } + + return nil + }) }); err != nil { return nil, err } @@ -176,24 +189,26 @@ func (lm *LeaseManager) AddResource(ctx context.Context, lease leases.Lease, r l return err } - topbkt := getBucket(lm.tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases, []byte(lease.ID)) - if topbkt == nil { - return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) - } - - keys, ref, err := parseLeaseResource(r) - if err != nil { - return err - } + return update(ctx, lm.db, func(tx *bolt.Tx) error { + topbkt := getBucket(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases, []byte(lease.ID)) + if topbkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) + } - bkt := topbkt - for _, key := range keys { - bkt, err = bkt.CreateBucketIfNotExists([]byte(key)) + keys, ref, err := parseLeaseResource(r) if err != nil { return err } - } - return bkt.Put([]byte(ref), nil) + + bkt := topbkt + for _, key := range keys { + bkt, err = bkt.CreateBucketIfNotExists([]byte(key)) + if err != nil { + return err + } + } + return bkt.Put([]byte(ref), nil) + }) } // DeleteResource dereferences the resource by the provided lease. @@ -203,28 +218,35 @@ func (lm *LeaseManager) DeleteResource(ctx context.Context, lease leases.Lease, return err } - topbkt := getBucket(lm.tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases, []byte(lease.ID)) - if topbkt == nil { - return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) - } + return update(ctx, lm.db, func(tx *bolt.Tx) error { + topbkt := getBucket(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases, []byte(lease.ID)) + if topbkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) + } - keys, ref, err := parseLeaseResource(r) - if err != nil { - return err - } + keys, ref, err := parseLeaseResource(r) + if err != nil { + return err + } + + bkt := topbkt + for _, key := range keys { + if bkt == nil { + break + } + bkt = bkt.Bucket([]byte(key)) + } - bkt := topbkt - for _, key := range keys { - if bkt == nil { - break + if bkt != nil { + if err := bkt.Delete([]byte(ref)); err != nil { + return err + } } - bkt = bkt.Bucket([]byte(key)) - } - if bkt == nil { + atomic.AddUint32(&lm.db.dirty, 1) + return nil - } - return bkt.Delete([]byte(ref)) + }) } // ListResources lists all the resources referenced by the lease. @@ -234,59 +256,66 @@ func (lm *LeaseManager) ListResources(ctx context.Context, lease leases.Lease) ( return nil, err } - topbkt := getBucket(lm.tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases, []byte(lease.ID)) - if topbkt == nil { - return nil, errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) - } - - rs := make([]leases.Resource, 0) + var rs []leases.Resource - // content resources - if cbkt := topbkt.Bucket(bucketKeyObjectContent); cbkt != nil { - if err := cbkt.ForEach(func(k, _ []byte) error { - rs = append(rs, leases.Resource{ - ID: string(k), - Type: string(bucketKeyObjectContent), - }) + if err := view(ctx, lm.db, func(tx *bolt.Tx) error { - return nil - }); err != nil { - return nil, err + topbkt := getBucket(tx, bucketKeyVersion, []byte(namespace), bucketKeyObjectLeases, []byte(lease.ID)) + if topbkt == nil { + return errors.Wrapf(errdefs.ErrNotFound, "lease %q", lease.ID) } - } - - // ingest resources - if lbkt := topbkt.Bucket(bucketKeyObjectIngests); lbkt != nil { - if err := lbkt.ForEach(func(k, _ []byte) error { - rs = append(rs, leases.Resource{ - ID: string(k), - Type: string(bucketKeyObjectIngests), - }) - return nil - }); err != nil { - return nil, err - } - } + // content resources + if cbkt := topbkt.Bucket(bucketKeyObjectContent); cbkt != nil { + if err := cbkt.ForEach(func(k, _ []byte) error { + rs = append(rs, leases.Resource{ + ID: string(k), + Type: string(bucketKeyObjectContent), + }) - // snapshot resources - if sbkt := topbkt.Bucket(bucketKeyObjectSnapshots); sbkt != nil { - if err := sbkt.ForEach(func(sk, sv []byte) error { - if sv != nil { return nil + }); err != nil { + return err } + } - snbkt := sbkt.Bucket(sk) - return snbkt.ForEach(func(k, _ []byte) error { + // ingest resources + if lbkt := topbkt.Bucket(bucketKeyObjectIngests); lbkt != nil { + if err := lbkt.ForEach(func(k, _ []byte) error { rs = append(rs, leases.Resource{ ID: string(k), - Type: fmt.Sprintf("%s/%s", bucketKeyObjectSnapshots, sk), + Type: string(bucketKeyObjectIngests), }) + return nil - }) - }); err != nil { - return nil, err + }); err != nil { + return err + } } + + // snapshot resources + if sbkt := topbkt.Bucket(bucketKeyObjectSnapshots); sbkt != nil { + if err := sbkt.ForEach(func(sk, sv []byte) error { + if sv != nil { + return nil + } + + snbkt := sbkt.Bucket(sk) + return snbkt.ForEach(func(k, _ []byte) error { + rs = append(rs, leases.Resource{ + ID: string(k), + Type: fmt.Sprintf("%s/%s", bucketKeyObjectSnapshots, sk), + }) + return nil + }) + }); err != nil { + return err + } + } + + return nil + }); err != nil { + return nil, err } return rs, nil } diff --git a/vendor/github.com/containerd/containerd/metadata/snapshot.go b/vendor/github.com/containerd/containerd/metadata/snapshot.go index 23976636f..4c38b41d7 100644 --- a/vendor/github.com/containerd/containerd/metadata/snapshot.go +++ b/vendor/github.com/containerd/containerd/metadata/snapshot.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/containerd/containerd/errdefs" @@ -517,9 +518,8 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error { } // Mark snapshotter as dirty for triggering garbage collection - s.db.dirtyL.Lock() + atomic.AddUint32(&s.db.dirty, 1) s.db.dirtySS[s.name] = struct{}{} - s.db.dirtyL.Unlock() return nil }) diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/blkio.go b/vendor/github.com/containerd/containerd/metrics/cgroups/blkio.go index feffaf0d1..1d1f0a999 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/blkio.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/blkio.go @@ -21,7 +21,7 @@ package cgroups import ( "strconv" - "github.com/containerd/cgroups" + v1 "github.com/containerd/containerd/metrics/types/v1" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -33,7 +33,7 @@ var blkioMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -46,7 +46,7 @@ var blkioMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -59,7 +59,7 @@ var blkioMetrics = []*metric{ unit: metrics.Bytes, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -72,7 +72,7 @@ var blkioMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -85,7 +85,7 @@ var blkioMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -98,7 +98,7 @@ var blkioMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -111,7 +111,7 @@ var blkioMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"op", "device", "major", "minor"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Blkio == nil { return nil } @@ -120,7 +120,7 @@ var blkioMetrics = []*metric{ }, } -func blkioValues(l []*cgroups.BlkIOEntry) []value { +func blkioValues(l []*v1.BlkIOEntry) []value { var out []value for _, e := range l { out = append(out, value{ diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/cpu.go b/vendor/github.com/containerd/containerd/metrics/cgroups/cpu.go index 1fe988ce1..686977b3d 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/cpu.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/cpu.go @@ -21,7 +21,7 @@ package cgroups import ( "strconv" - "github.com/containerd/cgroups" + v1 "github.com/containerd/containerd/metrics/types/v1" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -32,7 +32,7 @@ var cpuMetrics = []*metric{ help: "The total cpu time", unit: metrics.Nanoseconds, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } @@ -48,7 +48,7 @@ var cpuMetrics = []*metric{ help: "The total kernel cpu time", unit: metrics.Nanoseconds, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } @@ -64,7 +64,7 @@ var cpuMetrics = []*metric{ help: "The total user cpu time", unit: metrics.Nanoseconds, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } @@ -81,7 +81,7 @@ var cpuMetrics = []*metric{ unit: metrics.Nanoseconds, vt: prometheus.GaugeValue, labels: []string{"cpu"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } @@ -100,7 +100,7 @@ var cpuMetrics = []*metric{ help: "The total cpu throttle periods", unit: metrics.Total, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } @@ -116,7 +116,7 @@ var cpuMetrics = []*metric{ help: "The total cpu throttled periods", unit: metrics.Total, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } @@ -132,7 +132,7 @@ var cpuMetrics = []*metric{ help: "The total cpu throttled time", unit: metrics.Nanoseconds, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.CPU == nil { return nil } diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/hugetlb.go b/vendor/github.com/containerd/containerd/metrics/cgroups/hugetlb.go index ec173ae1c..fbd5d3ba7 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/hugetlb.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/hugetlb.go @@ -19,7 +19,7 @@ package cgroups import ( - "github.com/containerd/cgroups" + v1 "github.com/containerd/containerd/metrics/types/v1" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -31,7 +31,7 @@ var hugetlbMetrics = []*metric{ unit: metrics.Bytes, vt: prometheus.GaugeValue, labels: []string{"page"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Hugetlb == nil { return nil } @@ -51,7 +51,7 @@ var hugetlbMetrics = []*metric{ unit: metrics.Total, vt: prometheus.GaugeValue, labels: []string{"page"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Hugetlb == nil { return nil } @@ -71,7 +71,7 @@ var hugetlbMetrics = []*metric{ unit: metrics.Bytes, vt: prometheus.GaugeValue, labels: []string{"page"}, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Hugetlb == nil { return nil } diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/memory.go b/vendor/github.com/containerd/containerd/metrics/cgroups/memory.go index e90295fb3..9925f01e3 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/memory.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/memory.go @@ -19,7 +19,7 @@ package cgroups import ( - "github.com/containerd/cgroups" + v1 "github.com/containerd/containerd/metrics/types/v1" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -30,7 +30,7 @@ var memoryMetrics = []*metric{ help: "The cache amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -46,7 +46,7 @@ var memoryMetrics = []*metric{ help: "The rss amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -62,7 +62,7 @@ var memoryMetrics = []*metric{ help: "The rss_huge amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -78,7 +78,7 @@ var memoryMetrics = []*metric{ help: "The mapped_file amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -94,7 +94,7 @@ var memoryMetrics = []*metric{ help: "The dirty amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -110,7 +110,7 @@ var memoryMetrics = []*metric{ help: "The writeback amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -126,7 +126,7 @@ var memoryMetrics = []*metric{ help: "The pgpgin amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -142,7 +142,7 @@ var memoryMetrics = []*metric{ help: "The pgpgout amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -158,7 +158,7 @@ var memoryMetrics = []*metric{ help: "The pgfault amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -174,7 +174,7 @@ var memoryMetrics = []*metric{ help: "The pgmajfault amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -190,7 +190,7 @@ var memoryMetrics = []*metric{ help: "The inactive_anon amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -206,7 +206,7 @@ var memoryMetrics = []*metric{ help: "The active_anon amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -222,7 +222,7 @@ var memoryMetrics = []*metric{ help: "The inactive_file amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -238,7 +238,7 @@ var memoryMetrics = []*metric{ help: "The active_file amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -254,7 +254,7 @@ var memoryMetrics = []*metric{ help: "The unevictable amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -270,7 +270,7 @@ var memoryMetrics = []*metric{ help: "The hierarchical_memory_limit amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -286,7 +286,7 @@ var memoryMetrics = []*metric{ help: "The hierarchical_memsw_limit amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -302,7 +302,7 @@ var memoryMetrics = []*metric{ help: "The total_cache amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -318,7 +318,7 @@ var memoryMetrics = []*metric{ help: "The total_rss amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -334,7 +334,7 @@ var memoryMetrics = []*metric{ help: "The total_rss_huge amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -350,7 +350,7 @@ var memoryMetrics = []*metric{ help: "The total_mapped_file amount used", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -366,7 +366,7 @@ var memoryMetrics = []*metric{ help: "The total_dirty amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -382,7 +382,7 @@ var memoryMetrics = []*metric{ help: "The total_writeback amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -398,7 +398,7 @@ var memoryMetrics = []*metric{ help: "The total_pgpgin amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -414,7 +414,7 @@ var memoryMetrics = []*metric{ help: "The total_pgpgout amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -430,7 +430,7 @@ var memoryMetrics = []*metric{ help: "The total_pgfault amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -446,7 +446,7 @@ var memoryMetrics = []*metric{ help: "The total_pgmajfault amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -462,7 +462,7 @@ var memoryMetrics = []*metric{ help: "The total_inactive_anon amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -478,7 +478,7 @@ var memoryMetrics = []*metric{ help: "The total_active_anon amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -494,7 +494,7 @@ var memoryMetrics = []*metric{ help: "The total_inactive_file amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -510,7 +510,7 @@ var memoryMetrics = []*metric{ help: "The total_active_file amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -526,7 +526,7 @@ var memoryMetrics = []*metric{ help: "The total_unevictable amount", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -542,7 +542,7 @@ var memoryMetrics = []*metric{ help: "The usage failcnt", unit: metrics.Total, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -558,7 +558,7 @@ var memoryMetrics = []*metric{ help: "The memory limit", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -574,7 +574,7 @@ var memoryMetrics = []*metric{ help: "The memory maximum usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -590,7 +590,7 @@ var memoryMetrics = []*metric{ help: "The memory usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -606,7 +606,7 @@ var memoryMetrics = []*metric{ help: "The swap failcnt", unit: metrics.Total, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -622,7 +622,7 @@ var memoryMetrics = []*metric{ help: "The swap limit", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -638,7 +638,7 @@ var memoryMetrics = []*metric{ help: "The swap maximum usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -654,7 +654,7 @@ var memoryMetrics = []*metric{ help: "The swap usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -670,7 +670,7 @@ var memoryMetrics = []*metric{ help: "The kernel failcnt", unit: metrics.Total, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -686,7 +686,7 @@ var memoryMetrics = []*metric{ help: "The kernel limit", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -702,7 +702,7 @@ var memoryMetrics = []*metric{ help: "The kernel maximum usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -718,7 +718,7 @@ var memoryMetrics = []*metric{ help: "The kernel usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -734,7 +734,7 @@ var memoryMetrics = []*metric{ help: "The kerneltcp failcnt", unit: metrics.Total, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -750,7 +750,7 @@ var memoryMetrics = []*metric{ help: "The kerneltcp limit", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -766,7 +766,7 @@ var memoryMetrics = []*metric{ help: "The kerneltcp maximum usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } @@ -782,7 +782,7 @@ var memoryMetrics = []*metric{ help: "The kerneltcp usage", unit: metrics.Bytes, vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Memory == nil { return nil } diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/metric.go b/vendor/github.com/containerd/containerd/metrics/cgroups/metric.go index 02ad17776..c242bc85a 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/metric.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/metric.go @@ -19,7 +19,7 @@ package cgroups import ( - "github.com/containerd/cgroups" + v1 "github.com/containerd/containerd/metrics/types/v1" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -36,7 +36,7 @@ type metric struct { vt prometheus.ValueType labels []string // getValues returns the value and labels for the data - getValues func(stats *cgroups.Metrics) []value + getValues func(stats *v1.Metrics) []value } func (m *metric) desc(ns *metrics.Namespace) *prometheus.Desc { @@ -44,7 +44,7 @@ func (m *metric) desc(ns *metrics.Namespace) *prometheus.Desc { return ns.NewDesc(m.name, m.help, m.unit, append([]string{"container_id", "namespace"}, m.labels...)...) } -func (m *metric) collect(id, namespace string, stats *cgroups.Metrics, ns *metrics.Namespace, ch chan<- prometheus.Metric, block bool) { +func (m *metric) collect(id, namespace string, stats *v1.Metrics, ns *metrics.Namespace, ch chan<- prometheus.Metric, block bool) { values := m.getValues(stats) for _, v := range values { // block signals to block on the sending the metrics so none are missed diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/metrics.go b/vendor/github.com/containerd/containerd/metrics/cgroups/metrics.go index 5856d690e..23808731f 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/metrics.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/metrics.go @@ -25,6 +25,7 @@ import ( "github.com/containerd/cgroups" "github.com/containerd/containerd/log" + v1 "github.com/containerd/containerd/metrics/types/v1" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/runtime" "github.com/containerd/typeurl" @@ -114,7 +115,7 @@ func (c *collector) collect(t runtime.Task, ch chan<- prometheus.Metric, block b log.L.WithError(err).Errorf("unmarshal stats for %s", t.ID()) return } - s, ok := data.(*cgroups.Metrics) + s, ok := data.(*v1.Metrics) if !ok { log.L.WithError(err).Errorf("invalid metric type for %s", t.ID()) return diff --git a/vendor/github.com/containerd/containerd/metrics/cgroups/pids.go b/vendor/github.com/containerd/containerd/metrics/cgroups/pids.go index 82c294e25..c61fd6a97 100644 --- a/vendor/github.com/containerd/containerd/metrics/cgroups/pids.go +++ b/vendor/github.com/containerd/containerd/metrics/cgroups/pids.go @@ -19,7 +19,7 @@ package cgroups import ( - "github.com/containerd/cgroups" + v1 "github.com/containerd/containerd/metrics/types/v1" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -30,7 +30,7 @@ var pidMetrics = []*metric{ help: "The limit to the number of pids allowed", unit: metrics.Unit("limit"), vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Pids == nil { return nil } @@ -46,7 +46,7 @@ var pidMetrics = []*metric{ help: "The current number of pids", unit: metrics.Unit("current"), vt: prometheus.GaugeValue, - getValues: func(stats *cgroups.Metrics) []value { + getValues: func(stats *v1.Metrics) []value { if stats.Pids == nil { return nil } diff --git a/vendor/github.com/containerd/containerd/metrics/types/v1/types.go b/vendor/github.com/containerd/containerd/metrics/types/v1/types.go new file mode 100644 index 000000000..512f03595 --- /dev/null +++ b/vendor/github.com/containerd/containerd/metrics/types/v1/types.go @@ -0,0 +1,44 @@ +// +build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1 + +import "github.com/containerd/cgroups" + +type ( + // Metrics alias + Metrics = cgroups.Metrics + // BlkIOEntry alias + BlkIOEntry = cgroups.BlkIOEntry + // MemoryStat alias + MemoryStat = cgroups.MemoryStat + // CPUStat alias + CPUStat = cgroups.CPUStat + // CPUUsage alias + CPUUsage = cgroups.CPUUsage + // BlkIOStat alias + BlkIOStat = cgroups.BlkIOStat + // PidsStat alias + PidsStat = cgroups.PidsStat + // RdmaStat alias + RdmaStat = cgroups.RdmaStat + // RdmaEntry alias + RdmaEntry = cgroups.RdmaEntry + // HugetlbStat alias + HugetlbStat = cgroups.HugetlbStat +) diff --git a/vendor/github.com/containerd/containerd/namespaces/context.go b/vendor/github.com/containerd/containerd/namespaces/context.go index b4e988e7b..20596f09d 100644 --- a/vendor/github.com/containerd/containerd/namespaces/context.go +++ b/vendor/github.com/containerd/containerd/namespaces/context.go @@ -64,7 +64,7 @@ func Namespace(ctx context.Context) (string, bool) { return namespace, ok } -// NamespaceRequired returns the valid namepace from the context or an error. +// NamespaceRequired returns the valid namespace from the context or an error. func NamespaceRequired(ctx context.Context) (string, error) { namespace, ok := Namespace(ctx) if !ok || namespace == "" { diff --git a/vendor/github.com/containerd/containerd/oci/spec.go b/vendor/github.com/containerd/containerd/oci/spec.go index dad87411c..035bb7e7d 100644 --- a/vendor/github.com/containerd/containerd/oci/spec.go +++ b/vendor/github.com/containerd/containerd/oci/spec.go @@ -78,7 +78,7 @@ func generateDefaultSpecWithPlatform(ctx context.Context, platform, id string, s return err } -// ApplyOpts applys the options to the given spec, injecting data from the +// ApplyOpts applies the options to the given spec, injecting data from the // context, client and container instance. func ApplyOpts(ctx context.Context, client Client, c *containers.Container, s *Spec, opts ...SpecOpts) error { for _, o := range opts { diff --git a/vendor/github.com/containerd/containerd/pkg/process/deleted_state.go b/vendor/github.com/containerd/containerd/pkg/process/deleted_state.go index 95ad138e0..eb7baf737 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/deleted_state.go +++ b/vendor/github.com/containerd/containerd/pkg/process/deleted_state.go @@ -69,3 +69,7 @@ func (s *deletedState) SetExited(status int) { func (s *deletedState) Exec(ctx context.Context, path string, r *ExecConfig) (Process, error) { return nil, errors.Errorf("cannot exec in a deleted state") } + +func (s *deletedState) Status(ctx context.Context) (string, error) { + return "stopped", nil +} diff --git a/vendor/github.com/containerd/containerd/pkg/process/exec.go b/vendor/github.com/containerd/containerd/pkg/process/exec.go index 4175dcd5a..e77e79ae3 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/exec.go +++ b/vendor/github.com/containerd/containerd/pkg/process/exec.go @@ -96,7 +96,6 @@ func (e *execProcess) setExited(status int) { e.status = status e.exited = time.Now() e.parent.Platform.ShutdownConsole(context.Background(), e.console) - e.pid.set(StoppedPID) close(e.waitBlock) } @@ -147,7 +146,7 @@ func (e *execProcess) kill(ctx context.Context, sig uint32, _ bool) error { switch { case pid == 0: return errors.Wrap(errdefs.ErrFailedPrecondition, "process not created") - case pid < 0: + case !e.exited.IsZero(): return errors.Wrapf(errdefs.ErrNotFound, "process already finished") default: if err := unix.Kill(pid, syscall.Signal(sig)); err != nil { @@ -261,17 +260,5 @@ func (e *execProcess) Status(ctx context.Context) (string, error) { } e.mu.Lock() defer e.mu.Unlock() - // if we don't have a pid(pid=0) then the exec process has just been created - if e.pid.get() == 0 { - return "created", nil - } - if e.pid.get() == StoppedPID { - return "stopped", nil - } - // if we have a pid and it can be signaled, the process is running - if err := unix.Kill(e.pid.get(), 0); err == nil { - return "running", nil - } - // else if we have a pid but it can nolonger be signaled, it has stopped - return "stopped", nil + return e.execState.Status(ctx) } diff --git a/vendor/github.com/containerd/containerd/pkg/process/exec_state.go b/vendor/github.com/containerd/containerd/pkg/process/exec_state.go index a8b44bb8b..c97b4001b 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/exec_state.go +++ b/vendor/github.com/containerd/containerd/pkg/process/exec_state.go @@ -31,6 +31,7 @@ type execState interface { Delete(context.Context) error Kill(context.Context, uint32, bool) error SetExited(int) + Status(context.Context) (string, error) } type execCreatedState struct { @@ -82,6 +83,10 @@ func (s *execCreatedState) SetExited(status int) { } } +func (s *execCreatedState) Status(ctx context.Context) (string, error) { + return "created", nil +} + type execRunningState struct { p *execProcess } @@ -120,6 +125,10 @@ func (s *execRunningState) SetExited(status int) { } } +func (s *execRunningState) Status(ctx context.Context) (string, error) { + return "running", nil +} + type execStoppedState struct { p *execProcess } @@ -157,3 +166,7 @@ func (s *execStoppedState) Kill(ctx context.Context, sig uint32, all bool) error func (s *execStoppedState) SetExited(status int) { // no op } + +func (s *execStoppedState) Status(ctx context.Context) (string, error) { + return "stopped", nil +} diff --git a/vendor/github.com/containerd/containerd/pkg/process/init.go b/vendor/github.com/containerd/containerd/pkg/process/init.go index 7861bdd8b..e94a39a0d 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/init.go +++ b/vendor/github.com/containerd/containerd/pkg/process/init.go @@ -56,15 +56,17 @@ type Init struct { WorkDir string - id string - Bundle string - console console.Console - Platform stdio.Platform - io *processIO - runtime *runc.Runc + id string + Bundle string + console console.Console + Platform stdio.Platform + io *processIO + runtime *runc.Runc + // pausing preserves the pausing state. + pausing *atomicBool status int exited time.Time - pid safePid + pid int closers []io.Closer stdin io.Closer stdio stdio.Stdio @@ -97,6 +99,7 @@ func New(id string, runtime *runc.Runc, stdio stdio.Stdio) *Init { p := &Init{ id: id, runtime: runtime, + pausing: new(atomicBool), stdio: stdio, status: 0, waitBlock: make(chan struct{}), @@ -113,8 +116,6 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { pio *processIO pidFile = newPidFile(p.Bundle) ) - p.pid.Lock() - defer p.pid.Unlock() if r.Terminal { if socket, err = runc.NewTempConsoleSocket(); err != nil { @@ -170,7 +171,7 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { if err != nil { return errors.Wrap(err, "failed to retrieve OCI runtime container pid") } - p.pid.pid = pid + p.pid = pid return nil } @@ -216,7 +217,7 @@ func (p *Init) ID() string { // Pid of the process func (p *Init) Pid() int { - return p.pid.get() + return p.pid } // ExitStatus of the process @@ -237,17 +238,14 @@ func (p *Init) ExitedAt() time.Time { // Status of the process func (p *Init) Status(ctx context.Context) (string, error) { + if p.pausing.get() { + return "pausing", nil + } + p.mu.Lock() defer p.mu.Unlock() - c, err := p.runtime.State(ctx, p.id) - if err != nil { - if strings.Contains(err.Error(), "does not exist") { - return "stopped", nil - } - return "", p.runtimeError(err, "OCI runtime state failed") - } - return c.Status, nil + return p.initState.Status(ctx) } // Start the init process @@ -275,7 +273,6 @@ func (p *Init) setExited(status int) { p.exited = time.Now() p.status = status p.Platform.ShutdownConsole(context.Background(), p.console) - p.pid.set(StoppedPID) close(p.waitBlock) } diff --git a/vendor/github.com/containerd/containerd/pkg/process/init_state.go b/vendor/github.com/containerd/containerd/pkg/process/init_state.go index 9ec1d17be..81c6488a0 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/init_state.go +++ b/vendor/github.com/containerd/containerd/pkg/process/init_state.go @@ -37,6 +37,7 @@ type initState interface { Exec(context.Context, string, *ExecConfig) (Process, error) Kill(context.Context, uint32, bool) error SetExited(int) + Status(context.Context) (string, error) } type createdState struct { @@ -103,6 +104,10 @@ func (s *createdState) Exec(ctx context.Context, path string, r *ExecConfig) (Pr return s.p.exec(ctx, path, r) } +func (s *createdState) Status(ctx context.Context) (string, error) { + return "created", nil +} + type createdCheckpointState struct { p *Init opts *runc.RestoreOpts @@ -142,9 +147,6 @@ func (s *createdCheckpointState) Start(ctx context.Context) error { p := s.p sio := p.stdio - p.pid.Lock() - defer p.pid.Unlock() - var ( err error socket *runc.Socket @@ -184,7 +186,7 @@ func (s *createdCheckpointState) Start(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to retrieve OCI runtime container pid") } - p.pid.pid = pid + p.pid = pid return s.transition("running") } @@ -211,6 +213,10 @@ func (s *createdCheckpointState) Exec(ctx context.Context, path string, r *ExecC return nil, errors.Errorf("cannot exec in a created state") } +func (s *createdCheckpointState) Status(ctx context.Context) (string, error) { + return "created", nil +} + type runningState struct { p *Init } @@ -228,6 +234,13 @@ func (s *runningState) transition(name string) error { } func (s *runningState) Pause(ctx context.Context) error { + s.p.pausing.set(true) + // NOTE "pausing" will be returned in the short window + // after `transition("paused")`, before `pausing` is reset + // to false. That doesn't break the state machine, just + // delays the "paused" state a little bit. + defer s.p.pausing.set(false) + if err := s.p.runtime.Pause(ctx, s.p.id); err != nil { return s.p.runtimeError(err, "OCI runtime pause failed") } @@ -271,6 +284,10 @@ func (s *runningState) Exec(ctx context.Context, path string, r *ExecConfig) (Pr return s.p.exec(ctx, path, r) } +func (s *runningState) Status(ctx context.Context) (string, error) { + return "running", nil +} + type pausedState struct { p *Init } @@ -335,6 +352,10 @@ func (s *pausedState) Exec(ctx context.Context, path string, r *ExecConfig) (Pro return nil, errors.Errorf("cannot exec in a paused state") } +func (s *pausedState) Status(ctx context.Context) (string, error) { + return "paused", nil +} + type stoppedState struct { p *Init } @@ -387,3 +408,7 @@ func (s *stoppedState) SetExited(status int) { func (s *stoppedState) Exec(ctx context.Context, path string, r *ExecConfig) (Process, error) { return nil, errors.Errorf("cannot exec in a stopped state") } + +func (s *stoppedState) Status(ctx context.Context) (string, error) { + return "stopped", nil +} diff --git a/vendor/github.com/containerd/containerd/pkg/process/io.go b/vendor/github.com/containerd/containerd/pkg/process/io.go index 169f6c8e2..28a94a5ec 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/io.go +++ b/vendor/github.com/containerd/containerd/pkg/process/io.go @@ -40,7 +40,9 @@ import ( var bufPool = sync.Pool{ New: func() interface{} { - buffer := make([]byte, 32<<10) + // setting to 4096 to align with PIPE_BUF + // http://man7.org/linux/man-pages/man7/pipe.7.html + buffer := make([]byte, 4096) return &buffer }, } diff --git a/vendor/github.com/containerd/containerd/pkg/process/utils.go b/vendor/github.com/containerd/containerd/pkg/process/utils.go index b0ac6333c..3c4661770 100644 --- a/vendor/github.com/containerd/containerd/pkg/process/utils.go +++ b/vendor/github.com/containerd/containerd/pkg/process/utils.go @@ -27,6 +27,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/containerd/containerd/errdefs" @@ -38,8 +39,6 @@ import ( const ( // RuncRoot is the path to the root runc state directory RuncRoot = "/run/containerd/runc" - // StoppedPID is the pid assigned after a container has run and stopped - StoppedPID = -1 // InitPidFile name of the file that contains the init pid InitPidFile = "init.pid" ) @@ -56,10 +55,18 @@ func (s *safePid) get() int { return s.pid } -func (s *safePid) set(pid int) { - s.Lock() - s.pid = pid - s.Unlock() +type atomicBool int32 + +func (ab *atomicBool) set(b bool) { + if b { + atomic.StoreInt32((*int32)(ab), 1) + } else { + atomic.StoreInt32((*int32)(ab), 0) + } +} + +func (ab *atomicBool) get() bool { + return atomic.LoadInt32((*int32)(ab)) == 1 } // TODO(mlaventure): move to runc package? @@ -127,6 +134,7 @@ func checkKillError(err error) error { } if strings.Contains(err.Error(), "os: process already finished") || strings.Contains(err.Error(), "container not running") || + strings.Contains(strings.ToLower(err.Error()), "no such process") || err == unix.ESRCH { return errors.Wrapf(errdefs.ErrNotFound, "process already finished") } diff --git a/vendor/github.com/containerd/containerd/process.go b/vendor/github.com/containerd/containerd/process.go index 3a890b514..5b302569b 100644 --- a/vendor/github.com/containerd/containerd/process.go +++ b/vendor/github.com/containerd/containerd/process.go @@ -61,7 +61,7 @@ func NewExitStatus(code uint32, t time.Time, err error) *ExitStatus { } } -// ExitStatus encapsulates a process' exit status. +// ExitStatus encapsulates a process's exit status. // It is used by `Wait()` to return either a process exit code or an error type ExitStatus struct { code uint32 diff --git a/vendor/github.com/containerd/containerd/pull.go b/vendor/github.com/containerd/containerd/pull.go index 2520639df..b5c50d94f 100644 --- a/vendor/github.com/containerd/containerd/pull.go +++ b/vendor/github.com/containerd/containerd/pull.go @@ -70,6 +70,11 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (_ Ima } unpackWrapper, eg := u.handlerWrapper(ctx, &unpacks) defer func() { + if retErr != nil { + // Forcibly stop the unpacker if there is + // an error. + eg.Cancel() + } if err := eg.Wait(); err != nil { if retErr == nil { retErr = errors.Wrap(err, "unpack") diff --git a/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go b/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go index ce3da5524..ad8482fa3 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go @@ -136,7 +136,7 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R } func (r dockerFetcher) open(ctx context.Context, req *request, mediatype string, offset int64) (io.ReadCloser, error) { - req.header.Set("Accept", strings.Join([]string{mediatype, `*`}, ", ")) + req.header.Set("Accept", strings.Join([]string{mediatype, `*/*`}, ", ")) if offset > 0 { // Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints diff --git a/vendor/github.com/containerd/containerd/remotes/docker/pusher.go b/vendor/github.com/containerd/containerd/remotes/docker/pusher.go index 600868467..a96fe5a95 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/pusher.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/pusher.go @@ -80,7 +80,7 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten } req := p.request(host, http.MethodHead, existCheck...) - req.header.Set("Accept", strings.Join([]string{desc.MediaType, `*`}, ", ")) + req.header.Set("Accept", strings.Join([]string{desc.MediaType, `*/*`}, ", ")) log.G(ctx).WithField("url", req.String()).Debugf("checking and pushing to") diff --git a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index 8e65d8ccc..f126449c3 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -155,7 +155,7 @@ func NewResolver(options ResolverOptions) remotes.Resolver { images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageManifest, - ocispec.MediaTypeImageIndex, "*"}, ", ")) + ocispec.MediaTypeImageIndex, "*/*"}, ", ")) } else { resolveHeader["Accept"] = options.Headers["Accept"] delete(options.Headers, "Accept") diff --git a/vendor/github.com/containerd/containerd/remotes/handlers.go b/vendor/github.com/containerd/containerd/remotes/handlers.go index 0a243eda1..671fea106 100644 --- a/vendor/github.com/containerd/containerd/remotes/handlers.go +++ b/vendor/github.com/containerd/containerd/remotes/handlers.go @@ -62,21 +62,17 @@ func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string { } } - switch desc.MediaType { - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + switch mt := desc.MediaType; { + case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest: return "manifest-" + desc.Digest.String() - case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + case mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex: return "index-" + desc.Digest.String() - case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip, - images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip, - ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, - images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc: + case images.IsLayerType(mt): return "layer-" + desc.Digest.String() - case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: + case images.IsKnownConfig(mt): return "config-" + desc.Digest.String() default: - log.G(ctx).Warnf("reference for unknown type: %s", desc.MediaType) + log.G(ctx).Warnf("reference for unknown type: %s", mt) return "unknown-" + desc.Digest.String() } } diff --git a/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go b/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go index 0243c3986..fdaff5f9e 100644 --- a/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go +++ b/vendor/github.com/containerd/containerd/runtime/v1/linux/runtime.go @@ -50,7 +50,6 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - bolt "go.etcd.io/bbolt" "golang.org/x/sys/unix" ) @@ -112,13 +111,13 @@ func New(ic *plugin.InitContext) (interface{}, error) { } cfg := ic.Config.(*Config) r := &Runtime{ - root: ic.Root, - state: ic.State, - tasks: runtime.NewTaskList(), - db: m.(*metadata.DB), - address: ic.Address, - events: ic.Events, - config: cfg, + root: ic.Root, + state: ic.State, + tasks: runtime.NewTaskList(), + containers: metadata.NewContainerStore(m.(*metadata.DB)), + address: ic.Address, + events: ic.Events, + config: cfg, } tasks, err := r.restoreTasks(ic.Context) if err != nil { @@ -138,9 +137,9 @@ type Runtime struct { state string address string - tasks *runtime.TaskList - db *metadata.DB - events *exchange.Exchange + tasks *runtime.TaskList + containers containers.Store + events *exchange.Exchange config *Config } @@ -508,14 +507,8 @@ func (r *Runtime) getRuntime(ctx context.Context, ns, id string) (*runc.Runc, er } func (r *Runtime) getRuncOptions(ctx context.Context, id string) (*runctypes.RuncOptions, error) { - var container containers.Container - - if err := r.db.View(func(tx *bolt.Tx) error { - store := metadata.NewContainerStore(tx) - var err error - container, err = store.Get(ctx, id) - return err - }); err != nil { + container, err := r.containers.Get(ctx, id) + if err != nil { return nil, err } diff --git a/vendor/github.com/containerd/containerd/runtime/v1/linux/task.go b/vendor/github.com/containerd/containerd/runtime/v1/linux/task.go index 0970c3ea3..6e7f592b8 100644 --- a/vendor/github.com/containerd/containerd/runtime/v1/linux/task.go +++ b/vendor/github.com/containerd/containerd/runtime/v1/linux/task.go @@ -91,9 +91,12 @@ func (t *Task) PID() uint32 { // Delete the task and return the exit status func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { - rsp, err := t.shim.Delete(ctx, empty) - if err != nil && !errdefs.IsNotFound(err) { - return nil, errdefs.FromGRPC(err) + rsp, shimErr := t.shim.Delete(ctx, empty) + if shimErr != nil { + shimErr = errdefs.FromGRPC(shimErr) + if !errdefs.IsNotFound(shimErr) { + return nil, shimErr + } } t.tasks.Delete(ctx, t.id) if err := t.shim.KillShim(ctx); err != nil { @@ -102,6 +105,9 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { if err := t.bundle.Delete(); err != nil { log.G(ctx).WithError(err).Error("failed to delete bundle") } + if shimErr != nil { + return nil, shimErr + } t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{ ContainerID: t.id, ExitStatus: rsp.ExitStatus, diff --git a/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go b/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go index a722ea1c2..f3e1f4b7c 100644 --- a/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go +++ b/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go @@ -55,7 +55,7 @@ var ( empty = &ptypes.Empty{} bufPool = sync.Pool{ New: func() interface{} { - buffer := make([]byte, 32<<10) + buffer := make([]byte, 4096) return &buffer }, } @@ -217,7 +217,7 @@ func (s *Service) Delete(ctx context.Context, r *ptypes.Empty) (*shimapi.DeleteR return nil, err } if err := p.Delete(ctx); err != nil { - return nil, err + return nil, errdefs.ToGRPC(err) } s.mu.Lock() delete(s.processes, s.id) @@ -240,7 +240,7 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq return nil, err } if err := p.Delete(ctx); err != nil { - return nil, err + return nil, errdefs.ToGRPC(err) } s.mu.Lock() delete(s.processes, r.ID) diff --git a/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go b/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go index a4a4b90a3..65a8666e4 100644 --- a/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go +++ b/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go @@ -55,6 +55,7 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console io.CopyBuffer(epollConsole, in, *bp) // we need to shutdown epollConsole when pipe broken epollConsole.Shutdown(p.epoller.CloseConsole) + epollConsole.Close() }() } @@ -73,9 +74,8 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console p := bufPool.Get().(*[]byte) defer bufPool.Put(p) io.CopyBuffer(outw, epollConsole, *p) - epollConsole.Close() - outr.Close() outw.Close() + outr.Close() wg.Done() }() cwg.Wait() diff --git a/vendor/github.com/containerd/containerd/runtime/v2/manager.go b/vendor/github.com/containerd/containerd/runtime/v2/manager.go index 5bd986641..0e110f7bf 100644 --- a/vendor/github.com/containerd/containerd/runtime/v2/manager.go +++ b/vendor/github.com/containerd/containerd/runtime/v2/manager.go @@ -33,7 +33,6 @@ import ( "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/runtime" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - bolt "go.etcd.io/bbolt" ) // Config for the v2 runtime @@ -69,13 +68,15 @@ func init() { if err != nil { return nil, err } - return New(ic.Context, ic.Root, ic.State, ic.Address, ic.TTRPCAddress, ic.Events, m.(*metadata.DB)) + cs := metadata.NewContainerStore(m.(*metadata.DB)) + + return New(ic.Context, ic.Root, ic.State, ic.Address, ic.TTRPCAddress, ic.Events, cs) }, }) } // New task manager for v2 shims -func New(ctx context.Context, root, state, containerdAddress, containerdTTRPCAddress string, events *exchange.Exchange, db *metadata.DB) (*TaskManager, error) { +func New(ctx context.Context, root, state, containerdAddress, containerdTTRPCAddress string, events *exchange.Exchange, cs containers.Store) (*TaskManager, error) { for _, d := range []string{root, state} { if err := os.MkdirAll(d, 0711); err != nil { return nil, err @@ -88,7 +89,7 @@ func New(ctx context.Context, root, state, containerdAddress, containerdTTRPCAdd containerdTTRPCAddress: containerdTTRPCAddress, tasks: runtime.NewTaskList(), events: events, - db: db, + containers: cs, } if err := m.loadExistingTasks(ctx); err != nil { return nil, err @@ -103,9 +104,9 @@ type TaskManager struct { containerdAddress string containerdTTRPCAddress string - tasks *runtime.TaskList - events *exchange.Exchange - db *metadata.DB + tasks *runtime.TaskList + events *exchange.Exchange + containers containers.Store } // ID of the task manager @@ -278,13 +279,8 @@ func (m *TaskManager) loadTasks(ctx context.Context) error { } func (m *TaskManager) container(ctx context.Context, id string) (*containers.Container, error) { - var container containers.Container - if err := m.db.View(func(tx *bolt.Tx) error { - store := metadata.NewContainerStore(tx) - var err error - container, err = store.Get(ctx, id) - return err - }); err != nil { + container, err := m.containers.Get(ctx, id) + if err != nil { return nil, err } return &container, nil diff --git a/vendor/github.com/containerd/containerd/runtime/v2/shim.go b/vendor/github.com/containerd/containerd/runtime/v2/shim.go index 972f8222f..47e927437 100644 --- a/vendor/github.com/containerd/containerd/runtime/v2/shim.go +++ b/vendor/github.com/containerd/containerd/runtime/v2/shim.go @@ -222,11 +222,14 @@ func (s *shim) Close() error { } func (s *shim) Delete(ctx context.Context) (*runtime.Exit, error) { - response, err := s.task.Delete(ctx, &task.DeleteRequest{ + response, shimErr := s.task.Delete(ctx, &task.DeleteRequest{ ID: s.ID(), }) - if err != nil && !errdefs.IsNotFound(err) { - return nil, errdefs.FromGRPC(err) + if shimErr != nil { + shimErr = errdefs.FromGRPC(shimErr) + if !errdefs.IsNotFound(shimErr) { + return nil, shimErr + } } // remove self from the runtime task list // this seems dirty but it cleans up the API across runtimes, tasks, and the service @@ -238,6 +241,9 @@ func (s *shim) Delete(ctx context.Context) (*runtime.Exit, error) { if err := s.bundle.Delete(); err != nil { log.G(ctx).WithError(err).Error("failed to delete bundle") } + if shimErr != nil { + return nil, shimErr + } return &runtime.Exit{ Status: response.ExitStatus, Timestamp: response.ExitedAt, diff --git a/vendor/github.com/containerd/containerd/services/containers/local.go b/vendor/github.com/containerd/containerd/services/containers/local.go index 7b1a24b8f..b1336494c 100644 --- a/vendor/github.com/containerd/containerd/services/containers/local.go +++ b/vendor/github.com/containerd/containerd/services/containers/local.go @@ -48,8 +48,11 @@ func init() { if err != nil { return nil, err } + + db := m.(*metadata.DB) return &local{ - db: m.(*metadata.DB), + Store: metadata.NewContainerStore(db), + db: db, publisher: ic.Events, }, nil }, @@ -57,6 +60,7 @@ func init() { } type local struct { + containers.Store db *metadata.DB publisher events.Publisher } @@ -66,8 +70,8 @@ var _ api.ContainersClient = &local{} func (l *local) Get(ctx context.Context, req *api.GetContainerRequest, _ ...grpc.CallOption) (*api.GetContainerResponse, error) { var resp api.GetContainerResponse - return &resp, errdefs.ToGRPC(l.withStoreView(ctx, func(ctx context.Context, store containers.Store) error { - container, err := store.Get(ctx, req.ID) + return &resp, errdefs.ToGRPC(l.withStoreView(ctx, func(ctx context.Context) error { + container, err := l.Store.Get(ctx, req.ID) if err != nil { return err } @@ -80,8 +84,8 @@ func (l *local) Get(ctx context.Context, req *api.GetContainerRequest, _ ...grpc func (l *local) List(ctx context.Context, req *api.ListContainersRequest, _ ...grpc.CallOption) (*api.ListContainersResponse, error) { var resp api.ListContainersResponse - return &resp, errdefs.ToGRPC(l.withStoreView(ctx, func(ctx context.Context, store containers.Store) error { - containers, err := store.List(ctx, req.Filters...) + return &resp, errdefs.ToGRPC(l.withStoreView(ctx, func(ctx context.Context) error { + containers, err := l.Store.List(ctx, req.Filters...) if err != nil { return err } @@ -94,8 +98,8 @@ func (l *local) ListStream(ctx context.Context, req *api.ListContainersRequest, stream := &localStream{ ctx: ctx, } - return stream, errdefs.ToGRPC(l.withStoreView(ctx, func(ctx context.Context, store containers.Store) error { - containers, err := store.List(ctx, req.Filters...) + return stream, errdefs.ToGRPC(l.withStoreView(ctx, func(ctx context.Context) error { + containers, err := l.Store.List(ctx, req.Filters...) if err != nil { return err } @@ -107,10 +111,10 @@ func (l *local) ListStream(ctx context.Context, req *api.ListContainersRequest, func (l *local) Create(ctx context.Context, req *api.CreateContainerRequest, _ ...grpc.CallOption) (*api.CreateContainerResponse, error) { var resp api.CreateContainerResponse - if err := l.withStoreUpdate(ctx, func(ctx context.Context, store containers.Store) error { + if err := l.withStoreUpdate(ctx, func(ctx context.Context) error { container := containerFromProto(&req.Container) - created, err := store.Create(ctx, container) + created, err := l.Store.Create(ctx, container) if err != nil { return err } @@ -144,13 +148,13 @@ func (l *local) Update(ctx context.Context, req *api.UpdateContainerRequest, _ . container = containerFromProto(&req.Container) ) - if err := l.withStoreUpdate(ctx, func(ctx context.Context, store containers.Store) error { + if err := l.withStoreUpdate(ctx, func(ctx context.Context) error { var fieldpaths []string if req.UpdateMask != nil && len(req.UpdateMask.Paths) > 0 { fieldpaths = append(fieldpaths, req.UpdateMask.Paths...) } - updated, err := store.Update(ctx, container, fieldpaths...) + updated, err := l.Store.Update(ctx, container, fieldpaths...) if err != nil { return err } @@ -174,8 +178,8 @@ func (l *local) Update(ctx context.Context, req *api.UpdateContainerRequest, _ . } func (l *local) Delete(ctx context.Context, req *api.DeleteContainerRequest, _ ...grpc.CallOption) (*ptypes.Empty, error) { - if err := l.withStoreUpdate(ctx, func(ctx context.Context, store containers.Store) error { - return store.Delete(ctx, req.ID) + if err := l.withStoreUpdate(ctx, func(ctx context.Context) error { + return l.Store.Delete(ctx, req.ID) }); err != nil { return &ptypes.Empty{}, errdefs.ToGRPC(err) } @@ -189,15 +193,17 @@ func (l *local) Delete(ctx context.Context, req *api.DeleteContainerRequest, _ . return &ptypes.Empty{}, nil } -func (l *local) withStore(ctx context.Context, fn func(ctx context.Context, store containers.Store) error) func(tx *bolt.Tx) error { - return func(tx *bolt.Tx) error { return fn(ctx, metadata.NewContainerStore(tx)) } +func (l *local) withStore(ctx context.Context, fn func(ctx context.Context) error) func(tx *bolt.Tx) error { + return func(tx *bolt.Tx) error { + return fn(metadata.WithTransactionContext(ctx, tx)) + } } -func (l *local) withStoreView(ctx context.Context, fn func(ctx context.Context, store containers.Store) error) error { +func (l *local) withStoreView(ctx context.Context, fn func(ctx context.Context) error) error { return l.db.View(l.withStore(ctx, fn)) } -func (l *local) withStoreUpdate(ctx context.Context, fn func(ctx context.Context, store containers.Store) error) error { +func (l *local) withStoreUpdate(ctx context.Context, fn func(ctx context.Context) error) error { return l.db.Update(l.withStore(ctx, fn)) } diff --git a/vendor/github.com/containerd/containerd/services/leases/local.go b/vendor/github.com/containerd/containerd/services/leases/local.go index fcc621d4d..f942ba45f 100644 --- a/vendor/github.com/containerd/containerd/services/leases/local.go +++ b/vendor/github.com/containerd/containerd/services/leases/local.go @@ -24,7 +24,6 @@ import ( "github.com/containerd/containerd/metadata" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/services" - bolt "go.etcd.io/bbolt" ) func init() { @@ -44,8 +43,8 @@ func init() { return nil, err } return &local{ - db: m.(*metadata.DB), - gc: g.(gcScheduler), + Manager: metadata.NewLeaseManager(m.(*metadata.DB)), + gc: g.(gcScheduler), }, nil }, }) @@ -56,22 +55,10 @@ type gcScheduler interface { } type local struct { - db *metadata.DB + leases.Manager gc gcScheduler } -func (l *local) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) { - var lease leases.Lease - if err := l.db.Update(func(tx *bolt.Tx) error { - var err error - lease, err = metadata.NewLeaseManager(tx).Create(ctx, opts...) - return err - }); err != nil { - return leases.Lease{}, err - } - return lease, nil -} - func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error { var do leases.DeleteOptions for _, opt := range opts { @@ -80,9 +67,7 @@ func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.D } } - if err := l.db.Update(func(tx *bolt.Tx) error { - return metadata.NewLeaseManager(tx).Delete(ctx, lease) - }); err != nil { + if err := l.Manager.Delete(ctx, lease); err != nil { return err } @@ -95,39 +80,3 @@ func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.D return nil } - -func (l *local) List(ctx context.Context, filters ...string) ([]leases.Lease, error) { - var ll []leases.Lease - if err := l.db.View(func(tx *bolt.Tx) error { - var err error - ll, err = metadata.NewLeaseManager(tx).List(ctx, filters...) - return err - }); err != nil { - return nil, err - } - return ll, nil -} - -func (l *local) AddResource(ctx context.Context, lease leases.Lease, r leases.Resource) error { - return l.db.Update(func(tx *bolt.Tx) error { - return metadata.NewLeaseManager(tx).AddResource(ctx, lease, r) - }) -} - -func (l *local) DeleteResource(ctx context.Context, lease leases.Lease, r leases.Resource) error { - return l.db.Update(func(tx *bolt.Tx) error { - return metadata.NewLeaseManager(tx).DeleteResource(ctx, lease, r) - }) -} - -func (l *local) ListResources(ctx context.Context, lease leases.Lease) ([]leases.Resource, error) { - var rs []leases.Resource - if err := l.db.View(func(tx *bolt.Tx) error { - var err error - rs, err = metadata.NewLeaseManager(tx).ListResources(ctx, lease) - return err - }); err != nil { - return nil, err - } - return rs, nil -} diff --git a/vendor/github.com/containerd/containerd/services/tasks/local.go b/vendor/github.com/containerd/containerd/services/tasks/local.go index fc59936de..c93421d1f 100644 --- a/vendor/github.com/containerd/containerd/services/tasks/local.go +++ b/vendor/github.com/containerd/containerd/services/tasks/local.go @@ -51,7 +51,6 @@ import ( ptypes "github.com/gogo/protobuf/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -101,14 +100,14 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) { monitor = runtime.NewNoopMonitor() } - cs := m.(*metadata.DB).ContentStore() + db := m.(*metadata.DB) l := &local{ - runtimes: runtimes, - db: m.(*metadata.DB), - store: cs, - publisher: ic.Events, - monitor: monitor.(runtime.TaskMonitor), - v2Runtime: v2r.(*v2.TaskManager), + runtimes: runtimes, + containers: metadata.NewContainerStore(db), + store: db.ContentStore(), + publisher: ic.Events, + monitor: monitor.(runtime.TaskMonitor), + v2Runtime: v2r.(*v2.TaskManager), } for _, r := range runtimes { tasks, err := r.Tasks(ic.Context, true) @@ -123,10 +122,10 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) { } type local struct { - runtimes map[string]runtime.PlatformRuntime - db *metadata.DB - store content.Store - publisher events.Publisher + runtimes map[string]runtime.PlatformRuntime + containers containers.Store + store content.Store + publisher events.Publisher monitor runtime.TaskMonitor v2Runtime *v2.TaskManager @@ -242,7 +241,7 @@ func (l *local) Delete(ctx context.Context, r *api.DeleteTaskRequest, _ ...grpc. } exit, err := t.Delete(ctx) if err != nil { - return nil, err + return nil, errdefs.ToGRPC(err) } return &api.DeleteResponse{ ExitStatus: exit.Status, @@ -258,7 +257,7 @@ func (l *local) DeleteProcess(ctx context.Context, r *api.DeleteProcessRequest, } process, err := t.Process(ctx, r.ExecID) if err != nil { - return nil, err + return nil, errdefs.ToGRPC(err) } exit, err := process.Delete(ctx) if err != nil { @@ -647,12 +646,8 @@ func (l *local) writeContent(ctx context.Context, mediaType, ref string, r io.Re func (l *local) getContainer(ctx context.Context, id string) (*containers.Container, error) { var container containers.Container - if err := l.db.View(func(tx *bolt.Tx) error { - store := metadata.NewContainerStore(tx) - var err error - container, err = store.Get(ctx, id) - return err - }); err != nil { + container, err := l.containers.Get(ctx, id) + if err != nil { return nil, errdefs.ToGRPC(err) } return &container, nil diff --git a/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go index 7716a591e..3568c0cee 100644 --- a/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go @@ -297,7 +297,7 @@ func Remove(ctx context.Context, key string) (string, snapshots.Kind, error) { } if err := readSnapshot(sbkt, &id, &si); err != nil { - errors.Wrapf(err, "failed to read snapshot %s", key) + return errors.Wrapf(err, "failed to read snapshot %s", key) } if pbkt != nil { diff --git a/vendor/github.com/containerd/containerd/unpacker.go b/vendor/github.com/containerd/containerd/unpacker.go index 790c06c8d..6955129c7 100644 --- a/vendor/github.com/containerd/containerd/unpacker.go +++ b/vendor/github.com/containerd/containerd/unpacker.go @@ -186,8 +186,32 @@ func (u *unpacker) unpack(ctx context.Context, config ocispec.Descriptor, layers return nil } -func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(images.Handler) images.Handler, *errgroup.Group) { - eg, uctx := errgroup.WithContext(uctx) +type errGroup struct { + *errgroup.Group + cancel context.CancelFunc +} + +func newErrGroup(ctx context.Context) (*errGroup, context.Context) { + ctx, cancel := context.WithCancel(ctx) + eg, ctx := errgroup.WithContext(ctx) + return &errGroup{ + Group: eg, + cancel: cancel, + }, ctx +} + +func (e *errGroup) Cancel() { + e.cancel() +} + +func (e *errGroup) Wait() error { + err := e.Group.Wait() + e.cancel() + return err +} + +func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(images.Handler) images.Handler, *errGroup) { + eg, uctx := newErrGroup(uctx) return func(f images.Handler) images.Handler { var ( lock sync.Mutex @@ -204,12 +228,12 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im // one manifest to handle, and manifest list can be // safely skipped. // TODO: support multi-platform unpack. - switch desc.MediaType { - case images.MediaTypeDockerSchema1Manifest: + switch mt := desc.MediaType; { + case mt == images.MediaTypeDockerSchema1Manifest: lock.Lock() schema1 = true lock.Unlock() - case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest: lock.Lock() for _, child := range children { if child.MediaType == images.MediaTypeDockerSchema2Config || @@ -219,7 +243,7 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im layers = append(layers, child) } lock.Unlock() - case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: + case mt == images.MediaTypeDockerSchema2Config || mt == ocispec.MediaTypeImageConfig: lock.Lock() l := append([]ocispec.Descriptor{}, layers...) lock.Unlock() @@ -229,16 +253,24 @@ func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(im return u.unpack(uctx, desc, l) }) } - case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip, - images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip, - ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, - images.MediaTypeDockerSchema2LayerEnc, images.MediaTypeDockerSchema2LayerGzipEnc: + case images.IsLayerType(mt): lock.Lock() update := !schema1 lock.Unlock() if update { - u.updateCh <- desc + select { + case <-uctx.Done(): + // Do not send update if unpacker is not running. + default: + select { + case u.updateCh <- desc: + case <-uctx.Done(): + // Do not send update if unpacker is not running. + } + } + // Checking ctx.Done() prevents the case that unpacker + // exits unexpectedly, but update continues to be generated, + // and eventually fills up updateCh and blocks forever. } } return children, nil diff --git a/vendor/github.com/containerd/containerd/vendor.conf b/vendor/github.com/containerd/containerd/vendor.conf index 5f394e69c..92ef3ef9c 100644 --- a/vendor/github.com/containerd/containerd/vendor.conf +++ b/vendor/github.com/containerd/containerd/vendor.conf @@ -20,7 +20,7 @@ github.com/gogo/protobuf v1.2.1 github.com/gogo/googleapis v1.2.0 github.com/golang/protobuf v1.2.0 github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 # v1.0.1-59-g29686db -github.com/opencontainers/runc f4982d86f7fde0b6f953cc62ccc4022c519a10a9 # v1.0.0-rc8-32-gf4982d86 +github.com/opencontainers/runc d736ef14f0288d6993a1845745d6756cfc9ddd5a # v1.0.0-rc9 github.com/konsorten/go-windows-terminal-sequences v1.0.1 github.com/sirupsen/logrus v1.4.1 github.com/urfave/cli v1.22.0 @@ -51,7 +51,7 @@ github.com/cpuguy83/go-md2man v1.0.10 github.com/russross/blackfriday v1.5.2 # cri dependencies -github.com/containerd/cri f4d75d321c89b8d89bae570a7d2da1b3846c096f # release/1.3 +github.com/containerd/cri b1bef15fbeb6c6f0569b67322acfa74ca3597755 # release/1.3 github.com/containerd/go-cni 49fbd9b210f3c8ee3b7fd3cd797aabaf364627c1 github.com/containernetworking/cni v0.7.1 github.com/containernetworking/plugins v0.7.6 @@ -59,26 +59,26 @@ github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution 0d3efadf0154c2b8a4e7b6621fff9809655cc580 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528 -github.com/emicklei/go-restful v2.2.1 -github.com/google/gofuzz 24818f796faf91cd76ec7bddd72458fbced7a6c1 -github.com/json-iterator/go 1.1.5 +github.com/emicklei/go-restful v2.9.5 +github.com/google/gofuzz v1.0.0 +github.com/json-iterator/go v1.1.7 github.com/modern-go/reflect2 1.0.1 github.com/modern-go/concurrent 1.0.3 github.com/opencontainers/selinux v1.2.2 github.com/seccomp/libseccomp-golang v0.9.1 github.com/tchap/go-patricia v2.2.6 -golang.org/x/crypto 88737f569e3a9c7ab309cdc09a07fe7fc87233c3 -golang.org/x/oauth2 9f3314589c9a9136388751d9adae6b0ed400978a -golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 +golang.org/x/crypto 5c40567a22f818bd14a1ea7245dad9f8ef0691aa +golang.org/x/oauth2 0f29369cfe4552d0e4bcddc57cc75f4d7e672a33 +golang.org/x/time 85acf8d2951cb2a3bde7632f9ff273ef0379bcbd gopkg.in/inf.v0 v0.9.0 -gopkg.in/yaml.v2 v2.2.1 -k8s.io/api kubernetes-1.15.0 -k8s.io/apimachinery kubernetes-1.15.0 -k8s.io/apiserver kubernetes-1.15.0 -k8s.io/cri-api kubernetes-1.15.0 -k8s.io/client-go kubernetes-1.15.0 -k8s.io/klog v0.3.1 -k8s.io/kubernetes v1.15.0 +gopkg.in/yaml.v2 v2.2.2 +k8s.io/api kubernetes-1.16.0-rc.2 +k8s.io/apimachinery kubernetes-1.16.0-rc.2 +k8s.io/apiserver kubernetes-1.16.0-rc.2 +k8s.io/cri-api kubernetes-1.16.0-rc.2 +k8s.io/client-go kubernetes-1.16.0-rc.2 +k8s.io/klog v0.4.0 +k8s.io/kubernetes v1.16.0-rc.2 k8s.io/utils c2654d5206da6b7b6ace12841e8f359bb89b443c sigs.k8s.io/yaml v1.1.0 diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index b2874bf62..76c57c508 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -21,7 +21,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.2.0+unknown" + Version = "1.3.2+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/github.com/cpuguy83/go-md2man/LICENSE.md b/vendor/github.com/cpuguy83/go-md2man/LICENSE.md new file mode 100644 index 000000000..1cade6cef --- /dev/null +++ b/vendor/github.com/cpuguy83/go-md2man/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Brian Goff + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/cpuguy83/go-md2man/README.md b/vendor/github.com/cpuguy83/go-md2man/README.md new file mode 100644 index 000000000..29ed7c9e9 --- /dev/null +++ b/vendor/github.com/cpuguy83/go-md2man/README.md @@ -0,0 +1,21 @@ +go-md2man +========= + +** Work in Progress ** +This still needs a lot of help to be complete, or even usable! + +Uses blackfriday to process markdown into man pages. + +### Usage + +./md2man -in /path/to/markdownfile.md -out /manfile/output/path + +### How to contribute + +We use [dep](https://github.com/golang/dep/) for vendoring Go packages. +See dep documentation for how to update. + +### TODO + +- Needs oh so much testing love +- Look into blackfriday's 2.0 API diff --git a/vendor/github.com/cpuguy83/go-md2man/go.mod b/vendor/github.com/cpuguy83/go-md2man/go.mod new file mode 100644 index 000000000..052bd20ea --- /dev/null +++ b/vendor/github.com/cpuguy83/go-md2man/go.mod @@ -0,0 +1,5 @@ +module github.com/cpuguy83/go-md2man + +go 1.12 + +require github.com/russross/blackfriday v1.5.2 diff --git a/vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go b/vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go new file mode 100644 index 000000000..af62279a6 --- /dev/null +++ b/vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go @@ -0,0 +1,20 @@ +package md2man + +import ( + "github.com/russross/blackfriday" +) + +// Render converts a markdown document into a roff formatted document. +func Render(doc []byte) []byte { + renderer := RoffRenderer(0) + extensions := 0 + extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS + extensions |= blackfriday.EXTENSION_TABLES + extensions |= blackfriday.EXTENSION_FENCED_CODE + extensions |= blackfriday.EXTENSION_AUTOLINK + extensions |= blackfriday.EXTENSION_SPACE_HEADERS + extensions |= blackfriday.EXTENSION_FOOTNOTES + extensions |= blackfriday.EXTENSION_TITLEBLOCK + + return blackfriday.Markdown(doc, renderer, extensions) +} diff --git a/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go b/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go new file mode 100644 index 000000000..8c29ec687 --- /dev/null +++ b/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go @@ -0,0 +1,285 @@ +package md2man + +import ( + "bytes" + "fmt" + "html" + "strings" + + "github.com/russross/blackfriday" +) + +type roffRenderer struct { + ListCounters []int +} + +// RoffRenderer creates a new blackfriday Renderer for generating roff documents +// from markdown +func RoffRenderer(flags int) blackfriday.Renderer { + return &roffRenderer{} +} + +func (r *roffRenderer) GetFlags() int { + return 0 +} + +func (r *roffRenderer) TitleBlock(out *bytes.Buffer, text []byte) { + out.WriteString(".TH ") + + splitText := bytes.Split(text, []byte("\n")) + for i, line := range splitText { + line = bytes.TrimPrefix(line, []byte("% ")) + if i == 0 { + line = bytes.Replace(line, []byte("("), []byte("\" \""), 1) + line = bytes.Replace(line, []byte(")"), []byte("\" \""), 1) + } + line = append([]byte("\""), line...) + line = append(line, []byte("\" ")...) + out.Write(line) + } + out.WriteString("\n") + + // disable hyphenation + out.WriteString(".nh\n") + // disable justification (adjust text to left margin only) + out.WriteString(".ad l\n") +} + +func (r *roffRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) { + out.WriteString("\n.PP\n.RS\n\n.nf\n") + escapeSpecialChars(out, text) + out.WriteString("\n.fi\n.RE\n") +} + +func (r *roffRenderer) BlockQuote(out *bytes.Buffer, text []byte) { + out.WriteString("\n.PP\n.RS\n") + out.Write(text) + out.WriteString("\n.RE\n") +} + +func (r *roffRenderer) BlockHtml(out *bytes.Buffer, text []byte) { // nolint: golint + out.Write(text) +} + +func (r *roffRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) { + marker := out.Len() + + switch { + case marker == 0: + // This is the doc header + out.WriteString(".TH ") + case level == 1: + out.WriteString("\n\n.SH ") + case level == 2: + out.WriteString("\n.SH ") + default: + out.WriteString("\n.SS ") + } + + if !text() { + out.Truncate(marker) + return + } +} + +func (r *roffRenderer) HRule(out *bytes.Buffer) { + out.WriteString("\n.ti 0\n\\l'\\n(.lu'\n") +} + +func (r *roffRenderer) List(out *bytes.Buffer, text func() bool, flags int) { + marker := out.Len() + r.ListCounters = append(r.ListCounters, 1) + out.WriteString("\n.RS\n") + if !text() { + out.Truncate(marker) + return + } + r.ListCounters = r.ListCounters[:len(r.ListCounters)-1] + out.WriteString("\n.RE\n") +} + +func (r *roffRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) { + if flags&blackfriday.LIST_TYPE_ORDERED != 0 { + out.WriteString(fmt.Sprintf(".IP \"%3d.\" 5\n", r.ListCounters[len(r.ListCounters)-1])) + r.ListCounters[len(r.ListCounters)-1]++ + } else { + out.WriteString(".IP \\(bu 2\n") + } + out.Write(text) + out.WriteString("\n") +} + +func (r *roffRenderer) Paragraph(out *bytes.Buffer, text func() bool) { + marker := out.Len() + out.WriteString("\n.PP\n") + if !text() { + out.Truncate(marker) + return + } + if marker != 0 { + out.WriteString("\n") + } +} + +func (r *roffRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) { + out.WriteString("\n.TS\nallbox;\n") + + maxDelims := 0 + lines := strings.Split(strings.TrimRight(string(header), "\n")+"\n"+strings.TrimRight(string(body), "\n"), "\n") + for _, w := range lines { + curDelims := strings.Count(w, "\t") + if curDelims > maxDelims { + maxDelims = curDelims + } + } + out.Write([]byte(strings.Repeat("l ", maxDelims+1) + "\n")) + out.Write([]byte(strings.Repeat("l ", maxDelims+1) + ".\n")) + out.Write(header) + if len(header) > 0 { + out.Write([]byte("\n")) + } + + out.Write(body) + out.WriteString("\n.TE\n") +} + +func (r *roffRenderer) TableRow(out *bytes.Buffer, text []byte) { + if out.Len() > 0 { + out.WriteString("\n") + } + out.Write(text) +} + +func (r *roffRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, align int) { + if out.Len() > 0 { + out.WriteString("\t") + } + if len(text) == 0 { + text = []byte{' '} + } + out.Write([]byte("\\fB\\fC" + string(text) + "\\fR")) +} + +func (r *roffRenderer) TableCell(out *bytes.Buffer, text []byte, align int) { + if out.Len() > 0 { + out.WriteString("\t") + } + if len(text) > 30 { + text = append([]byte("T{\n"), text...) + text = append(text, []byte("\nT}")...) + } + if len(text) == 0 { + text = []byte{' '} + } + out.Write(text) +} + +func (r *roffRenderer) Footnotes(out *bytes.Buffer, text func() bool) { + +} + +func (r *roffRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) { + +} + +func (r *roffRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) { + out.WriteString("\n\\[la]") + out.Write(link) + out.WriteString("\\[ra]") +} + +func (r *roffRenderer) CodeSpan(out *bytes.Buffer, text []byte) { + out.WriteString("\\fB\\fC") + escapeSpecialChars(out, text) + out.WriteString("\\fR") +} + +func (r *roffRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) { + out.WriteString("\\fB") + out.Write(text) + out.WriteString("\\fP") +} + +func (r *roffRenderer) Emphasis(out *bytes.Buffer, text []byte) { + out.WriteString("\\fI") + out.Write(text) + out.WriteString("\\fP") +} + +func (r *roffRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) { +} + +func (r *roffRenderer) LineBreak(out *bytes.Buffer) { + out.WriteString("\n.br\n") +} + +func (r *roffRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { + out.Write(content) + r.AutoLink(out, link, 0) +} + +func (r *roffRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) { // nolint: golint + out.Write(tag) +} + +func (r *roffRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) { + out.WriteString("\\s+2") + out.Write(text) + out.WriteString("\\s-2") +} + +func (r *roffRenderer) StrikeThrough(out *bytes.Buffer, text []byte) { +} + +func (r *roffRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) { + +} + +func (r *roffRenderer) Entity(out *bytes.Buffer, entity []byte) { + out.WriteString(html.UnescapeString(string(entity))) +} + +func (r *roffRenderer) NormalText(out *bytes.Buffer, text []byte) { + escapeSpecialChars(out, text) +} + +func (r *roffRenderer) DocumentHeader(out *bytes.Buffer) { +} + +func (r *roffRenderer) DocumentFooter(out *bytes.Buffer) { +} + +func needsBackslash(c byte) bool { + for _, r := range []byte("-_&\\~") { + if c == r { + return true + } + } + return false +} + +func escapeSpecialChars(out *bytes.Buffer, text []byte) { + for i := 0; i < len(text); i++ { + // escape initial apostrophe or period + if len(text) >= 1 && (text[0] == '\'' || text[0] == '.') { + out.WriteString("\\&") + } + + // directly copy normal characters + org := i + + for i < len(text) && !needsBackslash(text[i]) { + i++ + } + if i > org { + out.Write(text[org:i]) + } + + // escape a character + if i >= len(text) { + break + } + out.WriteByte('\\') + out.WriteByte(text[i]) + } +} diff --git a/vendor/github.com/hashicorp/golang-lru/LICENSE b/vendor/github.com/hashicorp/golang-lru/LICENSE new file mode 100644 index 000000000..be2cc4dfb --- /dev/null +++ b/vendor/github.com/hashicorp/golang-lru/LICENSE @@ -0,0 +1,362 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/hashicorp/golang-lru/README.md b/vendor/github.com/hashicorp/golang-lru/README.md new file mode 100644 index 000000000..33e58cfaf --- /dev/null +++ b/vendor/github.com/hashicorp/golang-lru/README.md @@ -0,0 +1,25 @@ +golang-lru +========== + +This provides the `lru` package which implements a fixed-size +thread safe LRU cache. It is based on the cache in Groupcache. + +Documentation +============= + +Full docs are available on [Godoc](http://godoc.org/github.com/hashicorp/golang-lru) + +Example +======= + +Using the LRU is very simple: + +```go +l, _ := New(128) +for i := 0; i < 256; i++ { + l.Add(i, nil) +} +if l.Len() != 128 { + panic(fmt.Sprintf("bad len: %v", l.Len())) +} +``` diff --git a/vendor/github.com/hashicorp/golang-lru/go.mod b/vendor/github.com/hashicorp/golang-lru/go.mod new file mode 100644 index 000000000..8ad8826b3 --- /dev/null +++ b/vendor/github.com/hashicorp/golang-lru/go.mod @@ -0,0 +1,3 @@ +module github.com/hashicorp/golang-lru + +go 1.12 diff --git a/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go b/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go new file mode 100644 index 000000000..a86c8539e --- /dev/null +++ b/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go @@ -0,0 +1,177 @@ +package simplelru + +import ( + "container/list" + "errors" +) + +// EvictCallback is used to get a callback when a cache entry is evicted +type EvictCallback func(key interface{}, value interface{}) + +// LRU implements a non-thread safe fixed size LRU cache +type LRU struct { + size int + evictList *list.List + items map[interface{}]*list.Element + onEvict EvictCallback +} + +// entry is used to hold a value in the evictList +type entry struct { + key interface{} + value interface{} +} + +// NewLRU constructs an LRU of the given size +func NewLRU(size int, onEvict EvictCallback) (*LRU, error) { + if size <= 0 { + return nil, errors.New("Must provide a positive size") + } + c := &LRU{ + size: size, + evictList: list.New(), + items: make(map[interface{}]*list.Element), + onEvict: onEvict, + } + return c, nil +} + +// Purge is used to completely clear the cache. +func (c *LRU) Purge() { + for k, v := range c.items { + if c.onEvict != nil { + c.onEvict(k, v.Value.(*entry).value) + } + delete(c.items, k) + } + c.evictList.Init() +} + +// Add adds a value to the cache. Returns true if an eviction occurred. +func (c *LRU) Add(key, value interface{}) (evicted bool) { + // Check for existing item + if ent, ok := c.items[key]; ok { + c.evictList.MoveToFront(ent) + ent.Value.(*entry).value = value + return false + } + + // Add new item + ent := &entry{key, value} + entry := c.evictList.PushFront(ent) + c.items[key] = entry + + evict := c.evictList.Len() > c.size + // Verify size not exceeded + if evict { + c.removeOldest() + } + return evict +} + +// Get looks up a key's value from the cache. +func (c *LRU) Get(key interface{}) (value interface{}, ok bool) { + if ent, ok := c.items[key]; ok { + c.evictList.MoveToFront(ent) + if ent.Value.(*entry) == nil { + return nil, false + } + return ent.Value.(*entry).value, true + } + return +} + +// Contains checks if a key is in the cache, without updating the recent-ness +// or deleting it for being stale. +func (c *LRU) Contains(key interface{}) (ok bool) { + _, ok = c.items[key] + return ok +} + +// Peek returns the key value (or undefined if not found) without updating +// the "recently used"-ness of the key. +func (c *LRU) Peek(key interface{}) (value interface{}, ok bool) { + var ent *list.Element + if ent, ok = c.items[key]; ok { + return ent.Value.(*entry).value, true + } + return nil, ok +} + +// Remove removes the provided key from the cache, returning if the +// key was contained. +func (c *LRU) Remove(key interface{}) (present bool) { + if ent, ok := c.items[key]; ok { + c.removeElement(ent) + return true + } + return false +} + +// RemoveOldest removes the oldest item from the cache. +func (c *LRU) RemoveOldest() (key interface{}, value interface{}, ok bool) { + ent := c.evictList.Back() + if ent != nil { + c.removeElement(ent) + kv := ent.Value.(*entry) + return kv.key, kv.value, true + } + return nil, nil, false +} + +// GetOldest returns the oldest entry +func (c *LRU) GetOldest() (key interface{}, value interface{}, ok bool) { + ent := c.evictList.Back() + if ent != nil { + kv := ent.Value.(*entry) + return kv.key, kv.value, true + } + return nil, nil, false +} + +// Keys returns a slice of the keys in the cache, from oldest to newest. +func (c *LRU) Keys() []interface{} { + keys := make([]interface{}, len(c.items)) + i := 0 + for ent := c.evictList.Back(); ent != nil; ent = ent.Prev() { + keys[i] = ent.Value.(*entry).key + i++ + } + return keys +} + +// Len returns the number of items in the cache. +func (c *LRU) Len() int { + return c.evictList.Len() +} + +// Resize changes the cache size. +func (c *LRU) Resize(size int) (evicted int) { + diff := c.Len() - size + if diff < 0 { + diff = 0 + } + for i := 0; i < diff; i++ { + c.removeOldest() + } + c.size = size + return diff +} + +// removeOldest removes the oldest item from the cache. +func (c *LRU) removeOldest() { + ent := c.evictList.Back() + if ent != nil { + c.removeElement(ent) + } +} + +// removeElement is used to remove a given list element from the cache +func (c *LRU) removeElement(e *list.Element) { + c.evictList.Remove(e) + kv := e.Value.(*entry) + delete(c.items, kv.key) + if c.onEvict != nil { + c.onEvict(kv.key, kv.value) + } +} diff --git a/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go b/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go new file mode 100644 index 000000000..92d70934d --- /dev/null +++ b/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go @@ -0,0 +1,39 @@ +package simplelru + +// LRUCache is the interface for simple LRU cache. +type LRUCache interface { + // Adds a value to the cache, returns true if an eviction occurred and + // updates the "recently used"-ness of the key. + Add(key, value interface{}) bool + + // Returns key's value from the cache and + // updates the "recently used"-ness of the key. #value, isFound + Get(key interface{}) (value interface{}, ok bool) + + // Checks if a key exists in cache without updating the recent-ness. + Contains(key interface{}) (ok bool) + + // Returns key's value without updating the "recently used"-ness of the key. + Peek(key interface{}) (value interface{}, ok bool) + + // Removes a key from the cache. + Remove(key interface{}) bool + + // Removes the oldest entry from cache. + RemoveOldest() (interface{}, interface{}, bool) + + // Returns the oldest entry from the cache. #key, value, isFound + GetOldest() (interface{}, interface{}, bool) + + // Returns a slice of the keys in the cache, from oldest to newest. + Keys() []interface{} + + // Returns the number of items in the cache. + Len() int + + // Clears all cache entries. + Purge() + + // Resizes cache, returning number evicted + Resize(int) int +} diff --git a/vendor/github.com/opencontainers/runc/libcontainer/README.md b/vendor/github.com/opencontainers/runc/libcontainer/README.md index 1d7fa04c0..a791ca2d2 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/README.md +++ b/vendor/github.com/opencontainers/runc/libcontainer/README.md @@ -261,6 +261,7 @@ process := &libcontainer.Process{ Stdin: os.Stdin, Stdout: os.Stdout, Stderr: os.Stderr, + Init: true, } err := container.Run(process) diff --git a/vendor/github.com/opencontainers/runc/libcontainer/apparmor/apparmor.go b/vendor/github.com/opencontainers/runc/libcontainer/apparmor/apparmor.go index 7fff0627f..debfc1e48 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/apparmor/apparmor.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/apparmor/apparmor.go @@ -6,6 +6,8 @@ import ( "fmt" "io/ioutil" "os" + + "github.com/opencontainers/runc/libcontainer/utils" ) // IsEnabled returns true if apparmor is enabled for the host. @@ -19,7 +21,7 @@ func IsEnabled() bool { return false } -func setprocattr(attr, value string) error { +func setProcAttr(attr, value string) error { // Under AppArmor you can only change your own attr, so use /proc/self/ // instead of /proc// like libapparmor does path := fmt.Sprintf("/proc/self/attr/%s", attr) @@ -30,6 +32,10 @@ func setprocattr(attr, value string) error { } defer f.Close() + if err := utils.EnsureProcHandle(f); err != nil { + return err + } + _, err = fmt.Fprintf(f, "%s", value) return err } @@ -37,7 +43,7 @@ func setprocattr(attr, value string) error { // changeOnExec reimplements aa_change_onexec from libapparmor in Go func changeOnExec(name string) error { value := "exec " + name - if err := setprocattr("exec", value); err != nil { + if err := setProcAttr("exec", value); err != nil { return fmt.Errorf("apparmor failed to apply profile: %s", err) } return nil diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/blkio_device.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/blkio_device.go index e0f3ca165..fa195bf90 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/blkio_device.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/blkio_device.go @@ -59,3 +59,8 @@ func NewThrottleDevice(major, minor int64, rate uint64) *ThrottleDevice { func (td *ThrottleDevice) String() string { return fmt.Sprintf("%d:%d %d", td.Major, td.Minor, td.Rate) } + +// StringName formats the struct to be writable to the cgroup specific file +func (td *ThrottleDevice) StringName(name string) string { + return fmt.Sprintf("%d:%d %s=%d", td.Major, td.Minor, name, td.Rate) +} diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go index e15a662f5..58ed19c9e 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go @@ -119,4 +119,12 @@ type Resources struct { // Set class identifier for container's network packets NetClsClassid uint32 `json:"net_cls_classid_u"` + + // Used on cgroups v2: + + // CpuWeight sets a proportional bandwidth limit. + CpuWeight uint64 `json:"cpu_weight"` + + // CpuMax sets she maximum bandwidth limit (format: max period). + CpuMax string `json:"cpu_max"` } diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_windows.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_unsupported.go similarity index 89% rename from vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_windows.go rename to vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_unsupported.go index d74847b0d..c0c23d700 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_windows.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_unsupported.go @@ -1,3 +1,5 @@ +// +build !linux + package configs // TODO Windows: This can ultimately be entirely factored out on Windows as diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go index 7728522fe..24989e9f5 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -44,6 +44,7 @@ const ( Trap Allow Trace + Log ) // Operator is a comparison operator to be used when matching syscall arguments in Seccomp diff --git a/vendor/github.com/opencontainers/runc/libcontainer/devices/devices.go b/vendor/github.com/opencontainers/runc/libcontainer/devices/devices.go index 5e2ab0581..5dabe06ce 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/devices/devices.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/devices/devices.go @@ -7,11 +7,11 @@ import ( "path/filepath" "github.com/opencontainers/runc/libcontainer/configs" - "golang.org/x/sys/unix" ) var ( + // ErrNotADevice denotes that a file is not a valid linux device. ErrNotADevice = errors.New("not a device node") ) @@ -21,7 +21,8 @@ var ( ioutilReadDir = ioutil.ReadDir ) -// Given the path to a device and its cgroup_permissions(which cannot be easily queried) look up the information about a linux device and return that information as a Device struct. +// Given the path to a device and its cgroup_permissions(which cannot be easily queried) look up the +// information about a linux device and return that information as a Device struct. func DeviceFromPath(path, permissions string) (*configs.Device, error) { var stat unix.Stat_t err := unixLstat(path, &stat) @@ -60,25 +61,29 @@ func DeviceFromPath(path, permissions string) (*configs.Device, error) { }, nil } +// HostDevices returns all devices that can be found under /dev directory. func HostDevices() ([]*configs.Device, error) { - return getDevices("/dev") + return GetDevices("/dev") } -func getDevices(path string) ([]*configs.Device, error) { +// GetDevices recursively traverses a directory specified by path +// and returns all devices found there. +func GetDevices(path string) ([]*configs.Device, error) { files, err := ioutilReadDir(path) if err != nil { return nil, err } - out := []*configs.Device{} + var out []*configs.Device for _, f := range files { switch { case f.IsDir(): switch f.Name() { // ".lxc" & ".lxd-mounts" added to address https://github.com/lxc/lxd/issues/2825 - case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts": + // ".udev" added to address https://github.com/opencontainers/runc/issues/2093 + case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts", ".udev": continue default: - sub, err := getDevices(filepath.Join(path, f.Name())) + sub, err := GetDevices(filepath.Join(path, f.Name())) if err != nil { return nil, err } diff --git a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsenter_unsupported.go b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsenter_unsupported.go index ac701ca39..2459c6367 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsenter_unsupported.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsenter_unsupported.go @@ -1,5 +1,3 @@ // +build !linux !cgo package nsenter - -import "C" diff --git a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index 3b08c5e33..072656831 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -50,9 +50,6 @@ enum sync_t { #define JUMP_CHILD 0xA0 #define JUMP_INIT 0xA1 -/* JSON buffer. */ -#define JSON_MAX 4096 - /* Assume the stack grows down, so arguments should be above it. */ struct clone_t { /* @@ -148,11 +145,11 @@ static void write_log_with_info(const char *level, const char *function, int lin va_start(args, format); if (vsnprintf(message, sizeof(message), format, args) < 0) - return; - va_end(args); + goto done; - if (dprintf(logfd, "{\"level\":\"%s\", \"msg\": \"%s:%d %s\"}\n", level, function, line, message) < 0) - return; + dprintf(logfd, "{\"level\":\"%s\", \"msg\": \"%s:%d %s\"}\n", level, function, line, message); +done: + va_end(args); } #define write_log(level, fmt, ...) \ diff --git a/vendor/github.com/opencontainers/runc/libcontainer/seccomp/config.go b/vendor/github.com/opencontainers/runc/libcontainer/seccomp/config.go index ded5a6bbc..c32122798 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/seccomp/config.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/seccomp/config.go @@ -22,6 +22,7 @@ var actions = map[string]configs.Action{ "SCMP_ACT_TRAP": configs.Trap, "SCMP_ACT_ALLOW": configs.Allow, "SCMP_ACT_TRACE": configs.Trace, + "SCMP_ACT_LOG": configs.Log, } var archs = map[string]string{ diff --git a/vendor/github.com/opencontainers/runc/libcontainer/seccomp/seccomp_linux.go b/vendor/github.com/opencontainers/runc/libcontainer/seccomp/seccomp_linux.go index d99f3fe64..1b7a07118 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/seccomp/seccomp_linux.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/seccomp/seccomp_linux.go @@ -19,6 +19,7 @@ var ( actTrap = libseccomp.ActTrap actKill = libseccomp.ActKill actTrace = libseccomp.ActTrace.SetReturnCode(int16(unix.EPERM)) + actLog = libseccomp.ActLog actErrno = libseccomp.ActErrno.SetReturnCode(int16(unix.EPERM)) ) @@ -112,6 +113,8 @@ func getAction(act configs.Action) (libseccomp.ScmpAction, error) { return actAllow, nil case configs.Trace: return actTrace, nil + case configs.Log: + return actLog, nil default: return libseccomp.ActInvalid, fmt.Errorf("invalid action, cannot use in rule") } diff --git a/vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_64.go b/vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_64.go index 11c3faafb..e05e30adc 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_64.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_64.go @@ -1,5 +1,5 @@ // +build linux -// +build arm64 amd64 mips mipsle mips64 mips64le ppc ppc64 ppc64le s390x +// +build arm64 amd64 mips mipsle mips64 mips64le ppc ppc64 ppc64le riscv64 s390x package system diff --git a/vendor/github.com/opencontainers/runc/libcontainer/utils/cmsg.go b/vendor/github.com/opencontainers/runc/libcontainer/utils/cmsg.go new file mode 100644 index 000000000..c8a9364d5 --- /dev/null +++ b/vendor/github.com/opencontainers/runc/libcontainer/utils/cmsg.go @@ -0,0 +1,93 @@ +// +build linux + +package utils + +/* + * Copyright 2016, 2017 SUSE LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ( + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +// MaxSendfdLen is the maximum length of the name of a file descriptor being +// sent using SendFd. The name of the file handle returned by RecvFd will never +// be larger than this value. +const MaxNameLen = 4096 + +// oobSpace is the size of the oob slice required to store a single FD. Note +// that unix.UnixRights appears to make the assumption that fd is always int32, +// so sizeof(fd) = 4. +var oobSpace = unix.CmsgSpace(4) + +// RecvFd waits for a file descriptor to be sent over the given AF_UNIX +// socket. The file name of the remote file descriptor will be recreated +// locally (it is sent as non-auxiliary data in the same payload). +func RecvFd(socket *os.File) (*os.File, error) { + // For some reason, unix.Recvmsg uses the length rather than the capacity + // when passing the msg_controllen and other attributes to recvmsg. So we + // have to actually set the length. + name := make([]byte, MaxNameLen) + oob := make([]byte, oobSpace) + + sockfd := socket.Fd() + n, oobn, _, _, err := unix.Recvmsg(int(sockfd), name, oob, 0) + if err != nil { + return nil, err + } + + if n >= MaxNameLen || oobn != oobSpace { + return nil, fmt.Errorf("recvfd: incorrect number of bytes read (n=%d oobn=%d)", n, oobn) + } + + // Truncate. + name = name[:n] + oob = oob[:oobn] + + scms, err := unix.ParseSocketControlMessage(oob) + if err != nil { + return nil, err + } + if len(scms) != 1 { + return nil, fmt.Errorf("recvfd: number of SCMs is not 1: %d", len(scms)) + } + scm := scms[0] + + fds, err := unix.ParseUnixRights(&scm) + if err != nil { + return nil, err + } + if len(fds) != 1 { + return nil, fmt.Errorf("recvfd: number of fds is not 1: %d", len(fds)) + } + fd := uintptr(fds[0]) + + return os.NewFile(fd, string(name)), nil +} + +// SendFd sends a file descriptor over the given AF_UNIX socket. In +// addition, the file.Name() of the given file will also be sent as +// non-auxiliary data in the same payload (allowing to send contextual +// information for a file descriptor). +func SendFd(socket *os.File, name string, fd uintptr) error { + if len(name) >= MaxNameLen { + return fmt.Errorf("sendfd: filename too long: %s", name) + } + oob := unix.UnixRights(int(fd)) + return unix.Sendmsg(int(socket.Fd()), []byte(name), oob, nil, 0) +} diff --git a/vendor/github.com/opencontainers/runc/libcontainer/utils/utils.go b/vendor/github.com/opencontainers/runc/libcontainer/utils/utils.go new file mode 100644 index 000000000..40ccfaa1a --- /dev/null +++ b/vendor/github.com/opencontainers/runc/libcontainer/utils/utils.go @@ -0,0 +1,112 @@ +package utils + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + exitSignalOffset = 128 +) + +// ResolveRootfs ensures that the current working directory is +// not a symlink and returns the absolute path to the rootfs +func ResolveRootfs(uncleanRootfs string) (string, error) { + rootfs, err := filepath.Abs(uncleanRootfs) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(rootfs) +} + +// ExitStatus returns the correct exit status for a process based on if it +// was signaled or exited cleanly +func ExitStatus(status unix.WaitStatus) int { + if status.Signaled() { + return exitSignalOffset + int(status.Signal()) + } + return status.ExitStatus() +} + +// WriteJSON writes the provided struct v to w using standard json marshaling +func WriteJSON(w io.Writer, v interface{}) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + _, err = w.Write(data) + return err +} + +// CleanPath makes a path safe for use with filepath.Join. This is done by not +// only cleaning the path, but also (if the path is relative) adding a leading +// '/' and cleaning it (then removing the leading '/'). This ensures that a +// path resulting from prepending another path will always resolve to lexically +// be a subdirectory of the prefixed path. This is all done lexically, so paths +// that include symlinks won't be safe as a result of using CleanPath. +func CleanPath(path string) string { + // Deal with empty strings nicely. + if path == "" { + return "" + } + + // Ensure that all paths are cleaned (especially problematic ones like + // "/../../../../../" which can cause lots of issues). + path = filepath.Clean(path) + + // If the path isn't absolute, we need to do more processing to fix paths + // such as "../../../..//some/path". We also shouldn't convert absolute + // paths to relative ones. + if !filepath.IsAbs(path) { + path = filepath.Clean(string(os.PathSeparator) + path) + // This can't fail, as (by definition) all paths are relative to root. + path, _ = filepath.Rel(string(os.PathSeparator), path) + } + + // Clean the path again for good measure. + return filepath.Clean(path) +} + +// SearchLabels searches a list of key-value pairs for the provided key and +// returns the corresponding value. The pairs must be separated with '='. +func SearchLabels(labels []string, query string) string { + for _, l := range labels { + parts := strings.SplitN(l, "=", 2) + if len(parts) < 2 { + continue + } + if parts[0] == query { + return parts[1] + } + } + return "" +} + +// Annotations returns the bundle path and user defined annotations from the +// libcontainer state. We need to remove the bundle because that is a label +// added by libcontainer. +func Annotations(labels []string) (bundle string, userAnnotations map[string]string) { + userAnnotations = make(map[string]string) + for _, l := range labels { + parts := strings.SplitN(l, "=", 2) + if len(parts) < 2 { + continue + } + if parts[0] == "bundle" { + bundle = parts[1] + } else { + userAnnotations[parts[0]] = parts[1] + } + } + return +} + +func GetIntSize() int { + return int(unsafe.Sizeof(1)) +} diff --git a/vendor/github.com/opencontainers/runc/libcontainer/utils/utils_unix.go b/vendor/github.com/opencontainers/runc/libcontainer/utils/utils_unix.go new file mode 100644 index 000000000..1576f2d4a --- /dev/null +++ b/vendor/github.com/opencontainers/runc/libcontainer/utils/utils_unix.go @@ -0,0 +1,68 @@ +// +build !windows + +package utils + +import ( + "fmt" + "os" + "strconv" + + "golang.org/x/sys/unix" +) + +// EnsureProcHandle returns whether or not the given file handle is on procfs. +func EnsureProcHandle(fh *os.File) error { + var buf unix.Statfs_t + if err := unix.Fstatfs(int(fh.Fd()), &buf); err != nil { + return fmt.Errorf("ensure %s is on procfs: %v", fh.Name(), err) + } + if buf.Type != unix.PROC_SUPER_MAGIC { + return fmt.Errorf("%s is not on procfs", fh.Name()) + } + return nil +} + +// CloseExecFrom applies O_CLOEXEC to all file descriptors currently open for +// the process (except for those below the given fd value). +func CloseExecFrom(minFd int) error { + fdDir, err := os.Open("/proc/self/fd") + if err != nil { + return err + } + defer fdDir.Close() + + if err := EnsureProcHandle(fdDir); err != nil { + return err + } + + fdList, err := fdDir.Readdirnames(-1) + if err != nil { + return err + } + for _, fdStr := range fdList { + fd, err := strconv.Atoi(fdStr) + // Ignore non-numeric file names. + if err != nil { + continue + } + // Ignore descriptors lower than our specified minimum. + if fd < minFd { + continue + } + // Intentionally ignore errors from unix.CloseOnExec -- the cases where + // this might fail are basically file descriptors that have already + // been closed (including and especially the one that was created when + // ioutil.ReadDir did the "opendir" syscall). + unix.CloseOnExec(fd) + } + return nil +} + +// NewSockPair returns a new unix socket pair +func NewSockPair(name string) (parent *os.File, child *os.File, err error) { + fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + return os.NewFile(uintptr(fds[1]), name+"-p"), os.NewFile(uintptr(fds[0]), name+"-c"), nil +} diff --git a/vendor/github.com/opencontainers/runc/vendor.conf b/vendor/github.com/opencontainers/runc/vendor.conf index b0bfb1ef8..a29764cd7 100644 --- a/vendor/github.com/opencontainers/runc/vendor.conf +++ b/vendor/github.com/opencontainers/runc/vendor.conf @@ -1,26 +1,28 @@ # OCI runtime-spec. When updating this, make sure you use a version tag rather # than a commit ID so it's much more obvious what version of the spec we are # using. -github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 +github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 # v1.0.1-59-g29686db + # Core libcontainer functionality. -github.com/checkpoint-restore/go-criu v3.11 -github.com/mrunalp/fileutils ed869b029674c0e9ce4c0dfa781405c2d9946d08 -github.com/opencontainers/selinux v1.2.2 -github.com/seccomp/libseccomp-golang v0.9.1 -github.com/sirupsen/logrus 8bdbc7bcc01dcbb8ec23dc8a28e332258d25251f -github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 -github.com/vishvananda/netlink 1e2e08e8a2dcdacaae3f14ac44c5cfa31361f270 +github.com/checkpoint-restore/go-criu 17b0214f6c48980c45dc47ecb0cfd6d9e02df723 # v3.11 +github.com/mrunalp/fileutils 7d4729fb36185a7c1719923406c9d40e54fb93c7 +github.com/opencontainers/selinux 5215b1806f52b1fcc2070a8826c542c9d33cd3cf # v1.3.0 (+ CVE-2019-16884) +github.com/seccomp/libseccomp-golang 689e3c1541a84461afc49c1c87352a6cedf72e9c # v0.9.1 +github.com/sirupsen/logrus 8bdbc7bcc01dcbb8ec23dc8a28e332258d25251f # v1.4.1 +github.com/syndtr/gocapability d98352740cb2c55f81556b63d4a1ec64c5a319c2 +github.com/vishvananda/netlink 1e2e08e8a2dcdacaae3f14ac44c5cfa31361f270 + # systemd integration. -github.com/coreos/go-systemd v14 -github.com/coreos/pkg v3 -github.com/godbus/dbus v3 -github.com/golang/protobuf 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8 +github.com/coreos/go-systemd 95778dfbb74eb7e4dbaf43bf7d71809650ef8076 # v19 +github.com/godbus/dbus 2ff6f7ffd60f0f2410b3105864bdd12c7894f844 # v5.0.1 +github.com/golang/protobuf 925541529c1fa6821df4e44ce2723319eb2be768 # v1.0.0 + # Command-line interface. -github.com/cyphar/filepath-securejoin v0.2.1 -github.com/docker/go-units v0.2.0 -github.com/urfave/cli d53eb991652b1d438abdd34ce4bfa3ef1539108e -golang.org/x/sys 41f3e6584952bb034a481797859f6ab34b6803bd https://github.com/golang/sys +github.com/cyphar/filepath-securejoin a261ee33d7a517f054effbf451841abaafe3e0fd # v0.2.2 +github.com/docker/go-units 47565b4f722fb6ceae66b95f853feed578a4a51c # v0.3.3 +github.com/urfave/cli cfb38830724cc34fedffe9a2a29fb54fa9169cd1 # v1.20.0 +golang.org/x/sys 9eafafc0a87e0fd0aeeba439a4573537970c44c7 https://github.com/golang/sys # console dependencies -github.com/containerd/console 2748ece16665b45a47f884001d5831ec79703880 -github.com/pkg/errors v0.8.0 +github.com/containerd/console 0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f +github.com/pkg/errors ba968bfe8b2f7e042a574c888954fccecfa385b4 # v0.8.1 diff --git a/vendor/github.com/russross/blackfriday/LICENSE.txt b/vendor/github.com/russross/blackfriday/LICENSE.txt new file mode 100644 index 000000000..2885af360 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/LICENSE.txt @@ -0,0 +1,29 @@ +Blackfriday is distributed under the Simplified BSD License: + +> Copyright © 2011 Russ Ross +> All rights reserved. +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions +> are met: +> +> 1. Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> +> 2. Redistributions in binary form must reproduce the above +> copyright notice, this list of conditions and the following +> disclaimer in the documentation and/or other materials provided with +> the distribution. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +> FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +> COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +> INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +> BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +> LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +> CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +> LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +> ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +> POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/russross/blackfriday/README.md b/vendor/github.com/russross/blackfriday/README.md new file mode 100644 index 000000000..3c62e1375 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/README.md @@ -0,0 +1,369 @@ +Blackfriday +[![Build Status][BuildSVG]][BuildURL] +[![Godoc][GodocV2SVG]][GodocV2URL] +=========== + +Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It +is paranoid about its input (so you can safely feed it user-supplied +data), it is fast, it supports common extensions (tables, smart +punctuation substitutions, etc.), and it is safe for all utf-8 +(unicode) input. + +HTML output is currently supported, along with Smartypants +extensions. + +It started as a translation from C of [Sundown][3]. + + +Installation +------------ + +Blackfriday is compatible with any modern Go release. With Go and git installed: + + go get -u gopkg.in/russross/blackfriday.v2 + +will download, compile, and install the package into your `$GOPATH` directory +hierarchy. + + +Versions +-------- + +Currently maintained and recommended version of Blackfriday is `v2`. It's being +developed on its own branch: https://github.com/russross/blackfriday/tree/v2 and the +documentation is available at +https://godoc.org/gopkg.in/russross/blackfriday.v2. + +It is `go get`-able via [gopkg.in][6] at `gopkg.in/russross/blackfriday.v2`, +but we highly recommend using package management tool like [dep][7] or +[Glide][8] and make use of semantic versioning. With package management you +should import `github.com/russross/blackfriday` and specify that you're using +version 2.0.0. + +Version 2 offers a number of improvements over v1: + +* Cleaned up API +* A separate call to [`Parse`][4], which produces an abstract syntax tree for + the document +* Latest bug fixes +* Flexibility to easily add your own rendering extensions + +Potential drawbacks: + +* Our benchmarks show v2 to be slightly slower than v1. Currently in the + ballpark of around 15%. +* API breakage. If you can't afford modifying your code to adhere to the new API + and don't care too much about the new features, v2 is probably not for you. +* Several bug fixes are trailing behind and still need to be forward-ported to + v2. See issue [#348](https://github.com/russross/blackfriday/issues/348) for + tracking. + +If you are still interested in the legacy `v1`, you can import it from +`github.com/russross/blackfriday`. Documentation for the legacy v1 can be found +here: https://godoc.org/github.com/russross/blackfriday + +### Known issue with `dep` + +There is a known problem with using Blackfriday v1 _transitively_ and `dep`. +Currently `dep` prioritizes semver versions over anything else, and picks the +latest one, plus it does not apply a `[[constraint]]` specifier to transitively +pulled in packages. So if you're using something that uses Blackfriday v1, but +that something does not use `dep` yet, you will get Blackfriday v2 pulled in and +your first dependency will fail to build. + +There are couple of fixes for it, documented here: +https://github.com/golang/dep/blob/master/docs/FAQ.md#how-do-i-constrain-a-transitive-dependencys-version + +Meanwhile, `dep` team is working on a more general solution to the constraints +on transitive dependencies problem: https://github.com/golang/dep/issues/1124. + + +Usage +----- + +### v1 + +For basic usage, it is as simple as getting your input into a byte +slice and calling: + + output := blackfriday.MarkdownBasic(input) + +This renders it with no extensions enabled. To get a more useful +feature set, use this instead: + + output := blackfriday.MarkdownCommon(input) + +### v2 + +For the most sensible markdown processing, it is as simple as getting your input +into a byte slice and calling: + +```go +output := blackfriday.Run(input) +``` + +Your input will be parsed and the output rendered with a set of most popular +extensions enabled. If you want the most basic feature set, corresponding with +the bare Markdown specification, use: + +```go +output := blackfriday.Run(input, blackfriday.WithNoExtensions()) +``` + +### Sanitize untrusted content + +Blackfriday itself does nothing to protect against malicious content. If you are +dealing with user-supplied markdown, we recommend running Blackfriday's output +through HTML sanitizer such as [Bluemonday][5]. + +Here's an example of simple usage of Blackfriday together with Bluemonday: + +```go +import ( + "github.com/microcosm-cc/bluemonday" + "gopkg.in/russross/blackfriday.v2" +) + +// ... +unsafe := blackfriday.Run(input) +html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) +``` + +### Custom options, v1 + +If you want to customize the set of options, first get a renderer +(currently only the HTML output engine), then use it to +call the more general `Markdown` function. For examples, see the +implementations of `MarkdownBasic` and `MarkdownCommon` in +`markdown.go`. + +### Custom options, v2 + +If you want to customize the set of options, use `blackfriday.WithExtensions`, +`blackfriday.WithRenderer` and `blackfriday.WithRefOverride`. + +### `blackfriday-tool` + +You can also check out `blackfriday-tool` for a more complete example +of how to use it. Download and install it using: + + go get github.com/russross/blackfriday-tool + +This is a simple command-line tool that allows you to process a +markdown file using a standalone program. You can also browse the +source directly on github if you are just looking for some example +code: + +* + +Note that if you have not already done so, installing +`blackfriday-tool` will be sufficient to download and install +blackfriday in addition to the tool itself. The tool binary will be +installed in `$GOPATH/bin`. This is a statically-linked binary that +can be copied to wherever you need it without worrying about +dependencies and library versions. + +### Sanitized anchor names + +Blackfriday includes an algorithm for creating sanitized anchor names +corresponding to a given input text. This algorithm is used to create +anchors for headings when `EXTENSION_AUTO_HEADER_IDS` is enabled. The +algorithm has a specification, so that other packages can create +compatible anchor names and links to those anchors. + +The specification is located at https://godoc.org/github.com/russross/blackfriday#hdr-Sanitized_Anchor_Names. + +[`SanitizedAnchorName`](https://godoc.org/github.com/russross/blackfriday#SanitizedAnchorName) exposes this functionality, and can be used to +create compatible links to the anchor names generated by blackfriday. +This algorithm is also implemented in a small standalone package at +[`github.com/shurcooL/sanitized_anchor_name`](https://godoc.org/github.com/shurcooL/sanitized_anchor_name). It can be useful for clients +that want a small package and don't need full functionality of blackfriday. + + +Features +-------- + +All features of Sundown are supported, including: + +* **Compatibility**. The Markdown v1.0.3 test suite passes with + the `--tidy` option. Without `--tidy`, the differences are + mostly in whitespace and entity escaping, where blackfriday is + more consistent and cleaner. + +* **Common extensions**, including table support, fenced code + blocks, autolinks, strikethroughs, non-strict emphasis, etc. + +* **Safety**. Blackfriday is paranoid when parsing, making it safe + to feed untrusted user input without fear of bad things + happening. The test suite stress tests this and there are no + known inputs that make it crash. If you find one, please let me + know and send me the input that does it. + + NOTE: "safety" in this context means *runtime safety only*. In order to + protect yourself against JavaScript injection in untrusted content, see + [this example](https://github.com/russross/blackfriday#sanitize-untrusted-content). + +* **Fast processing**. It is fast enough to render on-demand in + most web applications without having to cache the output. + +* **Thread safety**. You can run multiple parsers in different + goroutines without ill effect. There is no dependence on global + shared state. + +* **Minimal dependencies**. Blackfriday only depends on standard + library packages in Go. The source code is pretty + self-contained, so it is easy to add to any project, including + Google App Engine projects. + +* **Standards compliant**. Output successfully validates using the + W3C validation tool for HTML 4.01 and XHTML 1.0 Transitional. + + +Extensions +---------- + +In addition to the standard markdown syntax, this package +implements the following extensions: + +* **Intra-word emphasis supression**. The `_` character is + commonly used inside words when discussing code, so having + markdown interpret it as an emphasis command is usually the + wrong thing. Blackfriday lets you treat all emphasis markers as + normal characters when they occur inside a word. + +* **Tables**. Tables can be created by drawing them in the input + using a simple syntax: + + ``` + Name | Age + --------|------ + Bob | 27 + Alice | 23 + ``` + +* **Fenced code blocks**. In addition to the normal 4-space + indentation to mark code blocks, you can explicitly mark them + and supply a language (to make syntax highlighting simple). Just + mark it like this: + + ``` go + func getTrue() bool { + return true + } + ``` + + You can use 3 or more backticks to mark the beginning of the + block, and the same number to mark the end of the block. + + To preserve classes of fenced code blocks while using the bluemonday + HTML sanitizer, use the following policy: + + ``` go + p := bluemonday.UGCPolicy() + p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code") + html := p.SanitizeBytes(unsafe) + ``` + +* **Definition lists**. A simple definition list is made of a single-line + term followed by a colon and the definition for that term. + + Cat + : Fluffy animal everyone likes + + Internet + : Vector of transmission for pictures of cats + + Terms must be separated from the previous definition by a blank line. + +* **Footnotes**. A marker in the text that will become a superscript number; + a footnote definition that will be placed in a list of footnotes at the + end of the document. A footnote looks like this: + + This is a footnote.[^1] + + [^1]: the footnote text. + +* **Autolinking**. Blackfriday can find URLs that have not been + explicitly marked as links and turn them into links. + +* **Strikethrough**. Use two tildes (`~~`) to mark text that + should be crossed out. + +* **Hard line breaks**. With this extension enabled (it is off by + default in the `MarkdownBasic` and `MarkdownCommon` convenience + functions), newlines in the input translate into line breaks in + the output. + +* **Smart quotes**. Smartypants-style punctuation substitution is + supported, turning normal double- and single-quote marks into + curly quotes, etc. + +* **LaTeX-style dash parsing** is an additional option, where `--` + is translated into `–`, and `---` is translated into + `—`. This differs from most smartypants processors, which + turn a single hyphen into an ndash and a double hyphen into an + mdash. + +* **Smart fractions**, where anything that looks like a fraction + is translated into suitable HTML (instead of just a few special + cases like most smartypant processors). For example, `4/5` + becomes `4⁄5`, which renders as + 4⁄5. + + +Other renderers +--------------- + +Blackfriday is structured to allow alternative rendering engines. Here +are a few of note: + +* [github_flavored_markdown](https://godoc.org/github.com/shurcooL/github_flavored_markdown): + provides a GitHub Flavored Markdown renderer with fenced code block + highlighting, clickable heading anchor links. + + It's not customizable, and its goal is to produce HTML output + equivalent to the [GitHub Markdown API endpoint](https://developer.github.com/v3/markdown/#render-a-markdown-document-in-raw-mode), + except the rendering is performed locally. + +* [markdownfmt](https://github.com/shurcooL/markdownfmt): like gofmt, + but for markdown. + +* [LaTeX output](https://bitbucket.org/ambrevar/blackfriday-latex): + renders output as LaTeX. + +* [bfchroma](https://github.com/Depado/bfchroma/): provides convenience + integration with the [Chroma](https://github.com/alecthomas/chroma) code + highlighting library. bfchroma is only compatible with v2 of Blackfriday and + provides a drop-in renderer ready to use with Blackfriday, as well as + options and means for further customization. + + +TODO +---- + +* More unit testing +* Improve Unicode support. It does not understand all Unicode + rules (about what constitutes a letter, a punctuation symbol, + etc.), so it may fail to detect word boundaries correctly in + some instances. It is safe on all UTF-8 input. + + +License +------- + +[Blackfriday is distributed under the Simplified BSD License](LICENSE.txt) + + + [1]: https://daringfireball.net/projects/markdown/ "Markdown" + [2]: https://golang.org/ "Go Language" + [3]: https://github.com/vmg/sundown "Sundown" + [4]: https://godoc.org/gopkg.in/russross/blackfriday.v2#Parse "Parse func" + [5]: https://github.com/microcosm-cc/bluemonday "Bluemonday" + [6]: https://labix.org/gopkg.in "gopkg.in" + [7]: https://github.com/golang/dep/ "dep" + [8]: https://github.com/Masterminds/glide "Glide" + + [BuildSVG]: https://travis-ci.org/russross/blackfriday.svg?branch=master + [BuildURL]: https://travis-ci.org/russross/blackfriday + [GodocV2SVG]: https://godoc.org/gopkg.in/russross/blackfriday.v2?status.svg + [GodocV2URL]: https://godoc.org/gopkg.in/russross/blackfriday.v2 diff --git a/vendor/github.com/russross/blackfriday/block.go b/vendor/github.com/russross/blackfriday/block.go new file mode 100644 index 000000000..45c21a6c2 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/block.go @@ -0,0 +1,1474 @@ +// +// Blackfriday Markdown Processor +// Available at http://github.com/russross/blackfriday +// +// Copyright © 2011 Russ Ross . +// Distributed under the Simplified BSD License. +// See README.md for details. +// + +// +// Functions to parse block-level elements. +// + +package blackfriday + +import ( + "bytes" + "strings" + "unicode" +) + +// Parse block-level data. +// Note: this function and many that it calls assume that +// the input buffer ends with a newline. +func (p *parser) block(out *bytes.Buffer, data []byte) { + if len(data) == 0 || data[len(data)-1] != '\n' { + panic("block input is missing terminating newline") + } + + // this is called recursively: enforce a maximum depth + if p.nesting >= p.maxNesting { + return + } + p.nesting++ + + // parse out one block-level construct at a time + for len(data) > 0 { + // prefixed header: + // + // # Header 1 + // ## Header 2 + // ... + // ###### Header 6 + if p.isPrefixHeader(data) { + data = data[p.prefixHeader(out, data):] + continue + } + + // block of preformatted HTML: + // + //
+ // ... + //
+ if data[0] == '<' { + if i := p.html(out, data, true); i > 0 { + data = data[i:] + continue + } + } + + // title block + // + // % stuff + // % more stuff + // % even more stuff + if p.flags&EXTENSION_TITLEBLOCK != 0 { + if data[0] == '%' { + if i := p.titleBlock(out, data, true); i > 0 { + data = data[i:] + continue + } + } + } + + // blank lines. note: returns the # of bytes to skip + if i := p.isEmpty(data); i > 0 { + data = data[i:] + continue + } + + // indented code block: + // + // func max(a, b int) int { + // if a > b { + // return a + // } + // return b + // } + if p.codePrefix(data) > 0 { + data = data[p.code(out, data):] + continue + } + + // fenced code block: + // + // ``` go info string here + // func fact(n int) int { + // if n <= 1 { + // return n + // } + // return n * fact(n-1) + // } + // ``` + if p.flags&EXTENSION_FENCED_CODE != 0 { + if i := p.fencedCodeBlock(out, data, true); i > 0 { + data = data[i:] + continue + } + } + + // horizontal rule: + // + // ------ + // or + // ****** + // or + // ______ + if p.isHRule(data) { + p.r.HRule(out) + var i int + for i = 0; data[i] != '\n'; i++ { + } + data = data[i:] + continue + } + + // block quote: + // + // > A big quote I found somewhere + // > on the web + if p.quotePrefix(data) > 0 { + data = data[p.quote(out, data):] + continue + } + + // table: + // + // Name | Age | Phone + // ------|-----|--------- + // Bob | 31 | 555-1234 + // Alice | 27 | 555-4321 + if p.flags&EXTENSION_TABLES != 0 { + if i := p.table(out, data); i > 0 { + data = data[i:] + continue + } + } + + // an itemized/unordered list: + // + // * Item 1 + // * Item 2 + // + // also works with + or - + if p.uliPrefix(data) > 0 { + data = data[p.list(out, data, 0):] + continue + } + + // a numbered/ordered list: + // + // 1. Item 1 + // 2. Item 2 + if p.oliPrefix(data) > 0 { + data = data[p.list(out, data, LIST_TYPE_ORDERED):] + continue + } + + // definition lists: + // + // Term 1 + // : Definition a + // : Definition b + // + // Term 2 + // : Definition c + if p.flags&EXTENSION_DEFINITION_LISTS != 0 { + if p.dliPrefix(data) > 0 { + data = data[p.list(out, data, LIST_TYPE_DEFINITION):] + continue + } + } + + // anything else must look like a normal paragraph + // note: this finds underlined headers, too + data = data[p.paragraph(out, data):] + } + + p.nesting-- +} + +func (p *parser) isPrefixHeader(data []byte) bool { + if data[0] != '#' { + return false + } + + if p.flags&EXTENSION_SPACE_HEADERS != 0 { + level := 0 + for level < 6 && data[level] == '#' { + level++ + } + if data[level] != ' ' { + return false + } + } + return true +} + +func (p *parser) prefixHeader(out *bytes.Buffer, data []byte) int { + level := 0 + for level < 6 && data[level] == '#' { + level++ + } + i := skipChar(data, level, ' ') + end := skipUntilChar(data, i, '\n') + skip := end + id := "" + if p.flags&EXTENSION_HEADER_IDS != 0 { + j, k := 0, 0 + // find start/end of header id + for j = i; j < end-1 && (data[j] != '{' || data[j+1] != '#'); j++ { + } + for k = j + 1; k < end && data[k] != '}'; k++ { + } + // extract header id iff found + if j < end && k < end { + id = string(data[j+2 : k]) + end = j + skip = k + 1 + for end > 0 && data[end-1] == ' ' { + end-- + } + } + } + for end > 0 && data[end-1] == '#' { + if isBackslashEscaped(data, end-1) { + break + } + end-- + } + for end > 0 && data[end-1] == ' ' { + end-- + } + if end > i { + if id == "" && p.flags&EXTENSION_AUTO_HEADER_IDS != 0 { + id = SanitizedAnchorName(string(data[i:end])) + } + work := func() bool { + p.inline(out, data[i:end]) + return true + } + p.r.Header(out, work, level, id) + } + return skip +} + +func (p *parser) isUnderlinedHeader(data []byte) int { + // test of level 1 header + if data[0] == '=' { + i := skipChar(data, 1, '=') + i = skipChar(data, i, ' ') + if data[i] == '\n' { + return 1 + } else { + return 0 + } + } + + // test of level 2 header + if data[0] == '-' { + i := skipChar(data, 1, '-') + i = skipChar(data, i, ' ') + if data[i] == '\n' { + return 2 + } else { + return 0 + } + } + + return 0 +} + +func (p *parser) titleBlock(out *bytes.Buffer, data []byte, doRender bool) int { + if data[0] != '%' { + return 0 + } + splitData := bytes.Split(data, []byte("\n")) + var i int + for idx, b := range splitData { + if !bytes.HasPrefix(b, []byte("%")) { + i = idx // - 1 + break + } + } + + data = bytes.Join(splitData[0:i], []byte("\n")) + p.r.TitleBlock(out, data) + + return len(data) +} + +func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int { + var i, j int + + // identify the opening tag + if data[0] != '<' { + return 0 + } + curtag, tagfound := p.htmlFindTag(data[1:]) + + // handle special cases + if !tagfound { + // check for an HTML comment + if size := p.htmlComment(out, data, doRender); size > 0 { + return size + } + + // check for an
tag + if size := p.htmlHr(out, data, doRender); size > 0 { + return size + } + + // check for HTML CDATA + if size := p.htmlCDATA(out, data, doRender); size > 0 { + return size + } + + // no special case recognized + return 0 + } + + // look for an unindented matching closing tag + // followed by a blank line + found := false + /* + closetag := []byte("\n") + j = len(curtag) + 1 + for !found { + // scan for a closing tag at the beginning of a line + if skip := bytes.Index(data[j:], closetag); skip >= 0 { + j += skip + len(closetag) + } else { + break + } + + // see if it is the only thing on the line + if skip := p.isEmpty(data[j:]); skip > 0 { + // see if it is followed by a blank line/eof + j += skip + if j >= len(data) { + found = true + i = j + } else { + if skip := p.isEmpty(data[j:]); skip > 0 { + j += skip + found = true + i = j + } + } + } + } + */ + + // if not found, try a second pass looking for indented match + // but not if tag is "ins" or "del" (following original Markdown.pl) + if !found && curtag != "ins" && curtag != "del" { + i = 1 + for i < len(data) { + i++ + for i < len(data) && !(data[i-1] == '<' && data[i] == '/') { + i++ + } + + if i+2+len(curtag) >= len(data) { + break + } + + j = p.htmlFindEnd(curtag, data[i-1:]) + + if j > 0 { + i += j - 1 + found = true + break + } + } + } + + if !found { + return 0 + } + + // the end of the block has been found + if doRender { + // trim newlines + end := i + for end > 0 && data[end-1] == '\n' { + end-- + } + p.r.BlockHtml(out, data[:end]) + } + + return i +} + +func (p *parser) renderHTMLBlock(out *bytes.Buffer, data []byte, start int, doRender bool) int { + // html block needs to end with a blank line + if i := p.isEmpty(data[start:]); i > 0 { + size := start + i + if doRender { + // trim trailing newlines + end := size + for end > 0 && data[end-1] == '\n' { + end-- + } + p.r.BlockHtml(out, data[:end]) + } + return size + } + return 0 +} + +// HTML comment, lax form +func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int { + i := p.inlineHTMLComment(out, data) + return p.renderHTMLBlock(out, data, i, doRender) +} + +// HTML CDATA section +func (p *parser) htmlCDATA(out *bytes.Buffer, data []byte, doRender bool) int { + const cdataTag = "') { + i++ + } + i++ + // no end-of-comment marker + if i >= len(data) { + return 0 + } + return p.renderHTMLBlock(out, data, i, doRender) +} + +// HR, which is the only self-closing block tag considered +func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int { + if data[0] != '<' || (data[1] != 'h' && data[1] != 'H') || (data[2] != 'r' && data[2] != 'R') { + return 0 + } + if data[3] != ' ' && data[3] != '/' && data[3] != '>' { + // not an
tag after all; at least not a valid one + return 0 + } + + i := 3 + for data[i] != '>' && data[i] != '\n' { + i++ + } + + if data[i] == '>' { + return p.renderHTMLBlock(out, data, i+1, doRender) + } + + return 0 +} + +func (p *parser) htmlFindTag(data []byte) (string, bool) { + i := 0 + for isalnum(data[i]) { + i++ + } + key := string(data[:i]) + if _, ok := blockTags[key]; ok { + return key, true + } + return "", false +} + +func (p *parser) htmlFindEnd(tag string, data []byte) int { + // assume data[0] == '<' && data[1] == '/' already tested + + // check if tag is a match + closetag := []byte("") + if !bytes.HasPrefix(data, closetag) { + return 0 + } + i := len(closetag) + + // check that the rest of the line is blank + skip := 0 + if skip = p.isEmpty(data[i:]); skip == 0 { + return 0 + } + i += skip + skip = 0 + + if i >= len(data) { + return i + } + + if p.flags&EXTENSION_LAX_HTML_BLOCKS != 0 { + return i + } + if skip = p.isEmpty(data[i:]); skip == 0 { + // following line must be blank + return 0 + } + + return i + skip +} + +func (*parser) isEmpty(data []byte) int { + // it is okay to call isEmpty on an empty buffer + if len(data) == 0 { + return 0 + } + + var i int + for i = 0; i < len(data) && data[i] != '\n'; i++ { + if data[i] != ' ' && data[i] != '\t' { + return 0 + } + } + return i + 1 +} + +func (*parser) isHRule(data []byte) bool { + i := 0 + + // skip up to three spaces + for i < 3 && data[i] == ' ' { + i++ + } + + // look at the hrule char + if data[i] != '*' && data[i] != '-' && data[i] != '_' { + return false + } + c := data[i] + + // the whole line must be the char or whitespace + n := 0 + for data[i] != '\n' { + switch { + case data[i] == c: + n++ + case data[i] != ' ': + return false + } + i++ + } + + return n >= 3 +} + +// isFenceLine checks if there's a fence line (e.g., ``` or ``` go) at the beginning of data, +// and returns the end index if so, or 0 otherwise. It also returns the marker found. +// If syntax is not nil, it gets set to the syntax specified in the fence line. +// A final newline is mandatory to recognize the fence line, unless newlineOptional is true. +func isFenceLine(data []byte, info *string, oldmarker string, newlineOptional bool) (end int, marker string) { + i, size := 0, 0 + + // skip up to three spaces + for i < len(data) && i < 3 && data[i] == ' ' { + i++ + } + + // check for the marker characters: ~ or ` + if i >= len(data) { + return 0, "" + } + if data[i] != '~' && data[i] != '`' { + return 0, "" + } + + c := data[i] + + // the whole line must be the same char or whitespace + for i < len(data) && data[i] == c { + size++ + i++ + } + + // the marker char must occur at least 3 times + if size < 3 { + return 0, "" + } + marker = string(data[i-size : i]) + + // if this is the end marker, it must match the beginning marker + if oldmarker != "" && marker != oldmarker { + return 0, "" + } + + // TODO(shurcooL): It's probably a good idea to simplify the 2 code paths here + // into one, always get the info string, and discard it if the caller doesn't care. + if info != nil { + infoLength := 0 + i = skipChar(data, i, ' ') + + if i >= len(data) { + if newlineOptional && i == len(data) { + return i, marker + } + return 0, "" + } + + infoStart := i + + if data[i] == '{' { + i++ + infoStart++ + + for i < len(data) && data[i] != '}' && data[i] != '\n' { + infoLength++ + i++ + } + + if i >= len(data) || data[i] != '}' { + return 0, "" + } + + // strip all whitespace at the beginning and the end + // of the {} block + for infoLength > 0 && isspace(data[infoStart]) { + infoStart++ + infoLength-- + } + + for infoLength > 0 && isspace(data[infoStart+infoLength-1]) { + infoLength-- + } + + i++ + } else { + for i < len(data) && !isverticalspace(data[i]) { + infoLength++ + i++ + } + } + + *info = strings.TrimSpace(string(data[infoStart : infoStart+infoLength])) + } + + i = skipChar(data, i, ' ') + if i >= len(data) || data[i] != '\n' { + if newlineOptional && i == len(data) { + return i, marker + } + return 0, "" + } + + return i + 1, marker // Take newline into account. +} + +// fencedCodeBlock returns the end index if data contains a fenced code block at the beginning, +// or 0 otherwise. It writes to out if doRender is true, otherwise it has no side effects. +// If doRender is true, a final newline is mandatory to recognize the fenced code block. +func (p *parser) fencedCodeBlock(out *bytes.Buffer, data []byte, doRender bool) int { + var infoString string + beg, marker := isFenceLine(data, &infoString, "", false) + if beg == 0 || beg >= len(data) { + return 0 + } + + var work bytes.Buffer + + for { + // safe to assume beg < len(data) + + // check for the end of the code block + newlineOptional := !doRender + fenceEnd, _ := isFenceLine(data[beg:], nil, marker, newlineOptional) + if fenceEnd != 0 { + beg += fenceEnd + break + } + + // copy the current line + end := skipUntilChar(data, beg, '\n') + 1 + + // did we reach the end of the buffer without a closing marker? + if end >= len(data) { + return 0 + } + + // verbatim copy to the working buffer + if doRender { + work.Write(data[beg:end]) + } + beg = end + } + + if doRender { + p.r.BlockCode(out, work.Bytes(), infoString) + } + + return beg +} + +func (p *parser) table(out *bytes.Buffer, data []byte) int { + var header bytes.Buffer + i, columns := p.tableHeader(&header, data) + if i == 0 { + return 0 + } + + var body bytes.Buffer + + for i < len(data) { + pipes, rowStart := 0, i + for ; data[i] != '\n'; i++ { + if data[i] == '|' { + pipes++ + } + } + + if pipes == 0 { + i = rowStart + break + } + + // include the newline in data sent to tableRow + i++ + p.tableRow(&body, data[rowStart:i], columns, false) + } + + p.r.Table(out, header.Bytes(), body.Bytes(), columns) + + return i +} + +// check if the specified position is preceded by an odd number of backslashes +func isBackslashEscaped(data []byte, i int) bool { + backslashes := 0 + for i-backslashes-1 >= 0 && data[i-backslashes-1] == '\\' { + backslashes++ + } + return backslashes&1 == 1 +} + +func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns []int) { + i := 0 + colCount := 1 + for i = 0; data[i] != '\n'; i++ { + if data[i] == '|' && !isBackslashEscaped(data, i) { + colCount++ + } + } + + // doesn't look like a table header + if colCount == 1 { + return + } + + // include the newline in the data sent to tableRow + header := data[:i+1] + + // column count ignores pipes at beginning or end of line + if data[0] == '|' { + colCount-- + } + if i > 2 && data[i-1] == '|' && !isBackslashEscaped(data, i-1) { + colCount-- + } + + columns = make([]int, colCount) + + // move on to the header underline + i++ + if i >= len(data) { + return + } + + if data[i] == '|' && !isBackslashEscaped(data, i) { + i++ + } + i = skipChar(data, i, ' ') + + // each column header is of form: / *:?-+:? *|/ with # dashes + # colons >= 3 + // and trailing | optional on last column + col := 0 + for data[i] != '\n' { + dashes := 0 + + if data[i] == ':' { + i++ + columns[col] |= TABLE_ALIGNMENT_LEFT + dashes++ + } + for data[i] == '-' { + i++ + dashes++ + } + if data[i] == ':' { + i++ + columns[col] |= TABLE_ALIGNMENT_RIGHT + dashes++ + } + for data[i] == ' ' { + i++ + } + + // end of column test is messy + switch { + case dashes < 3: + // not a valid column + return + + case data[i] == '|' && !isBackslashEscaped(data, i): + // marker found, now skip past trailing whitespace + col++ + i++ + for data[i] == ' ' { + i++ + } + + // trailing junk found after last column + if col >= colCount && data[i] != '\n' { + return + } + + case (data[i] != '|' || isBackslashEscaped(data, i)) && col+1 < colCount: + // something else found where marker was required + return + + case data[i] == '\n': + // marker is optional for the last column + col++ + + default: + // trailing junk found after last column + return + } + } + if col != colCount { + return + } + + p.tableRow(out, header, columns, true) + size = i + 1 + return +} + +func (p *parser) tableRow(out *bytes.Buffer, data []byte, columns []int, header bool) { + i, col := 0, 0 + var rowWork bytes.Buffer + + if data[i] == '|' && !isBackslashEscaped(data, i) { + i++ + } + + for col = 0; col < len(columns) && i < len(data); col++ { + for data[i] == ' ' { + i++ + } + + cellStart := i + + for (data[i] != '|' || isBackslashEscaped(data, i)) && data[i] != '\n' { + i++ + } + + cellEnd := i + + // skip the end-of-cell marker, possibly taking us past end of buffer + i++ + + for cellEnd > cellStart && data[cellEnd-1] == ' ' { + cellEnd-- + } + + var cellWork bytes.Buffer + p.inline(&cellWork, data[cellStart:cellEnd]) + + if header { + p.r.TableHeaderCell(&rowWork, cellWork.Bytes(), columns[col]) + } else { + p.r.TableCell(&rowWork, cellWork.Bytes(), columns[col]) + } + } + + // pad it out with empty columns to get the right number + for ; col < len(columns); col++ { + if header { + p.r.TableHeaderCell(&rowWork, nil, columns[col]) + } else { + p.r.TableCell(&rowWork, nil, columns[col]) + } + } + + // silently ignore rows with too many cells + + p.r.TableRow(out, rowWork.Bytes()) +} + +// returns blockquote prefix length +func (p *parser) quotePrefix(data []byte) int { + i := 0 + for i < 3 && data[i] == ' ' { + i++ + } + if data[i] == '>' { + if data[i+1] == ' ' { + return i + 2 + } + return i + 1 + } + return 0 +} + +// blockquote ends with at least one blank line +// followed by something without a blockquote prefix +func (p *parser) terminateBlockquote(data []byte, beg, end int) bool { + if p.isEmpty(data[beg:]) <= 0 { + return false + } + if end >= len(data) { + return true + } + return p.quotePrefix(data[end:]) == 0 && p.isEmpty(data[end:]) == 0 +} + +// parse a blockquote fragment +func (p *parser) quote(out *bytes.Buffer, data []byte) int { + var raw bytes.Buffer + beg, end := 0, 0 + for beg < len(data) { + end = beg + // Step over whole lines, collecting them. While doing that, check for + // fenced code and if one's found, incorporate it altogether, + // irregardless of any contents inside it + for data[end] != '\n' { + if p.flags&EXTENSION_FENCED_CODE != 0 { + if i := p.fencedCodeBlock(out, data[end:], false); i > 0 { + // -1 to compensate for the extra end++ after the loop: + end += i - 1 + break + } + } + end++ + } + end++ + + if pre := p.quotePrefix(data[beg:]); pre > 0 { + // skip the prefix + beg += pre + } else if p.terminateBlockquote(data, beg, end) { + break + } + + // this line is part of the blockquote + raw.Write(data[beg:end]) + beg = end + } + + var cooked bytes.Buffer + p.block(&cooked, raw.Bytes()) + p.r.BlockQuote(out, cooked.Bytes()) + return end +} + +// returns prefix length for block code +func (p *parser) codePrefix(data []byte) int { + if data[0] == ' ' && data[1] == ' ' && data[2] == ' ' && data[3] == ' ' { + return 4 + } + return 0 +} + +func (p *parser) code(out *bytes.Buffer, data []byte) int { + var work bytes.Buffer + + i := 0 + for i < len(data) { + beg := i + for data[i] != '\n' { + i++ + } + i++ + + blankline := p.isEmpty(data[beg:i]) > 0 + if pre := p.codePrefix(data[beg:i]); pre > 0 { + beg += pre + } else if !blankline { + // non-empty, non-prefixed line breaks the pre + i = beg + break + } + + // verbatim copy to the working buffeu + if blankline { + work.WriteByte('\n') + } else { + work.Write(data[beg:i]) + } + } + + // trim all the \n off the end of work + workbytes := work.Bytes() + eol := len(workbytes) + for eol > 0 && workbytes[eol-1] == '\n' { + eol-- + } + if eol != len(workbytes) { + work.Truncate(eol) + } + + work.WriteByte('\n') + + p.r.BlockCode(out, work.Bytes(), "") + + return i +} + +// returns unordered list item prefix +func (p *parser) uliPrefix(data []byte) int { + i := 0 + + // start with up to 3 spaces + for i < 3 && data[i] == ' ' { + i++ + } + + // need a *, +, or - followed by a space + if (data[i] != '*' && data[i] != '+' && data[i] != '-') || + data[i+1] != ' ' { + return 0 + } + return i + 2 +} + +// returns ordered list item prefix +func (p *parser) oliPrefix(data []byte) int { + i := 0 + + // start with up to 3 spaces + for i < 3 && data[i] == ' ' { + i++ + } + + // count the digits + start := i + for data[i] >= '0' && data[i] <= '9' { + i++ + } + + // we need >= 1 digits followed by a dot and a space + if start == i || data[i] != '.' || data[i+1] != ' ' { + return 0 + } + return i + 2 +} + +// returns definition list item prefix +func (p *parser) dliPrefix(data []byte) int { + i := 0 + + // need a : followed by a spaces + if data[i] != ':' || data[i+1] != ' ' { + return 0 + } + for data[i] == ' ' { + i++ + } + return i + 2 +} + +// parse ordered or unordered list block +func (p *parser) list(out *bytes.Buffer, data []byte, flags int) int { + i := 0 + flags |= LIST_ITEM_BEGINNING_OF_LIST + work := func() bool { + for i < len(data) { + skip := p.listItem(out, data[i:], &flags) + i += skip + + if skip == 0 || flags&LIST_ITEM_END_OF_LIST != 0 { + break + } + flags &= ^LIST_ITEM_BEGINNING_OF_LIST + } + return true + } + + p.r.List(out, work, flags) + return i +} + +// Parse a single list item. +// Assumes initial prefix is already removed if this is a sublist. +func (p *parser) listItem(out *bytes.Buffer, data []byte, flags *int) int { + // keep track of the indentation of the first line + itemIndent := 0 + for itemIndent < 3 && data[itemIndent] == ' ' { + itemIndent++ + } + + i := p.uliPrefix(data) + if i == 0 { + i = p.oliPrefix(data) + } + if i == 0 { + i = p.dliPrefix(data) + // reset definition term flag + if i > 0 { + *flags &= ^LIST_TYPE_TERM + } + } + if i == 0 { + // if in defnition list, set term flag and continue + if *flags&LIST_TYPE_DEFINITION != 0 { + *flags |= LIST_TYPE_TERM + } else { + return 0 + } + } + + // skip leading whitespace on first line + for data[i] == ' ' { + i++ + } + + // find the end of the line + line := i + for i > 0 && data[i-1] != '\n' { + i++ + } + + // get working buffer + var raw bytes.Buffer + + // put the first line into the working buffer + raw.Write(data[line:i]) + line = i + + // process the following lines + containsBlankLine := false + sublist := 0 + codeBlockMarker := "" + +gatherlines: + for line < len(data) { + i++ + + // find the end of this line + for data[i-1] != '\n' { + i++ + } + + // if it is an empty line, guess that it is part of this item + // and move on to the next line + if p.isEmpty(data[line:i]) > 0 { + containsBlankLine = true + raw.Write(data[line:i]) + line = i + continue + } + + // calculate the indentation + indent := 0 + for indent < 4 && line+indent < i && data[line+indent] == ' ' { + indent++ + } + + chunk := data[line+indent : i] + + if p.flags&EXTENSION_FENCED_CODE != 0 { + // determine if in or out of codeblock + // if in codeblock, ignore normal list processing + _, marker := isFenceLine(chunk, nil, codeBlockMarker, false) + if marker != "" { + if codeBlockMarker == "" { + // start of codeblock + codeBlockMarker = marker + } else { + // end of codeblock. + *flags |= LIST_ITEM_CONTAINS_BLOCK + codeBlockMarker = "" + } + } + // we are in a codeblock, write line, and continue + if codeBlockMarker != "" || marker != "" { + raw.Write(data[line+indent : i]) + line = i + continue gatherlines + } + } + + // evaluate how this line fits in + switch { + // is this a nested list item? + case (p.uliPrefix(chunk) > 0 && !p.isHRule(chunk)) || + p.oliPrefix(chunk) > 0 || + p.dliPrefix(chunk) > 0: + + if containsBlankLine { + // end the list if the type changed after a blank line + if indent <= itemIndent && + ((*flags&LIST_TYPE_ORDERED != 0 && p.uliPrefix(chunk) > 0) || + (*flags&LIST_TYPE_ORDERED == 0 && p.oliPrefix(chunk) > 0)) { + + *flags |= LIST_ITEM_END_OF_LIST + break gatherlines + } + *flags |= LIST_ITEM_CONTAINS_BLOCK + } + + // to be a nested list, it must be indented more + // if not, it is the next item in the same list + if indent <= itemIndent { + break gatherlines + } + + // is this the first item in the nested list? + if sublist == 0 { + sublist = raw.Len() + } + + // is this a nested prefix header? + case p.isPrefixHeader(chunk): + // if the header is not indented, it is not nested in the list + // and thus ends the list + if containsBlankLine && indent < 4 { + *flags |= LIST_ITEM_END_OF_LIST + break gatherlines + } + *flags |= LIST_ITEM_CONTAINS_BLOCK + + // anything following an empty line is only part + // of this item if it is indented 4 spaces + // (regardless of the indentation of the beginning of the item) + case containsBlankLine && indent < 4: + if *flags&LIST_TYPE_DEFINITION != 0 && i < len(data)-1 { + // is the next item still a part of this list? + next := i + for data[next] != '\n' { + next++ + } + for next < len(data)-1 && data[next] == '\n' { + next++ + } + if i < len(data)-1 && data[i] != ':' && data[next] != ':' { + *flags |= LIST_ITEM_END_OF_LIST + } + } else { + *flags |= LIST_ITEM_END_OF_LIST + } + break gatherlines + + // a blank line means this should be parsed as a block + case containsBlankLine: + *flags |= LIST_ITEM_CONTAINS_BLOCK + } + + containsBlankLine = false + + // add the line into the working buffer without prefix + raw.Write(data[line+indent : i]) + + line = i + } + + // If reached end of data, the Renderer.ListItem call we're going to make below + // is definitely the last in the list. + if line >= len(data) { + *flags |= LIST_ITEM_END_OF_LIST + } + + rawBytes := raw.Bytes() + + // render the contents of the list item + var cooked bytes.Buffer + if *flags&LIST_ITEM_CONTAINS_BLOCK != 0 && *flags&LIST_TYPE_TERM == 0 { + // intermediate render of block item, except for definition term + if sublist > 0 { + p.block(&cooked, rawBytes[:sublist]) + p.block(&cooked, rawBytes[sublist:]) + } else { + p.block(&cooked, rawBytes) + } + } else { + // intermediate render of inline item + if sublist > 0 { + p.inline(&cooked, rawBytes[:sublist]) + p.block(&cooked, rawBytes[sublist:]) + } else { + p.inline(&cooked, rawBytes) + } + } + + // render the actual list item + cookedBytes := cooked.Bytes() + parsedEnd := len(cookedBytes) + + // strip trailing newlines + for parsedEnd > 0 && cookedBytes[parsedEnd-1] == '\n' { + parsedEnd-- + } + p.r.ListItem(out, cookedBytes[:parsedEnd], *flags) + + return line +} + +// render a single paragraph that has already been parsed out +func (p *parser) renderParagraph(out *bytes.Buffer, data []byte) { + if len(data) == 0 { + return + } + + // trim leading spaces + beg := 0 + for data[beg] == ' ' { + beg++ + } + + // trim trailing newline + end := len(data) - 1 + + // trim trailing spaces + for end > beg && data[end-1] == ' ' { + end-- + } + + work := func() bool { + p.inline(out, data[beg:end]) + return true + } + p.r.Paragraph(out, work) +} + +func (p *parser) paragraph(out *bytes.Buffer, data []byte) int { + // prev: index of 1st char of previous line + // line: index of 1st char of current line + // i: index of cursor/end of current line + var prev, line, i int + + // keep going until we find something to mark the end of the paragraph + for i < len(data) { + // mark the beginning of the current line + prev = line + current := data[i:] + line = i + + // did we find a blank line marking the end of the paragraph? + if n := p.isEmpty(current); n > 0 { + // did this blank line followed by a definition list item? + if p.flags&EXTENSION_DEFINITION_LISTS != 0 { + if i < len(data)-1 && data[i+1] == ':' { + return p.list(out, data[prev:], LIST_TYPE_DEFINITION) + } + } + + p.renderParagraph(out, data[:i]) + return i + n + } + + // an underline under some text marks a header, so our paragraph ended on prev line + if i > 0 { + if level := p.isUnderlinedHeader(current); level > 0 { + // render the paragraph + p.renderParagraph(out, data[:prev]) + + // ignore leading and trailing whitespace + eol := i - 1 + for prev < eol && data[prev] == ' ' { + prev++ + } + for eol > prev && data[eol-1] == ' ' { + eol-- + } + + // render the header + // this ugly double closure avoids forcing variables onto the heap + work := func(o *bytes.Buffer, pp *parser, d []byte) func() bool { + return func() bool { + pp.inline(o, d) + return true + } + }(out, p, data[prev:eol]) + + id := "" + if p.flags&EXTENSION_AUTO_HEADER_IDS != 0 { + id = SanitizedAnchorName(string(data[prev:eol])) + } + + p.r.Header(out, work, level, id) + + // find the end of the underline + for data[i] != '\n' { + i++ + } + return i + } + } + + // if the next line starts a block of HTML, then the paragraph ends here + if p.flags&EXTENSION_LAX_HTML_BLOCKS != 0 { + if data[i] == '<' && p.html(out, current, false) > 0 { + // rewind to before the HTML block + p.renderParagraph(out, data[:i]) + return i + } + } + + // if there's a prefixed header or a horizontal rule after this, paragraph is over + if p.isPrefixHeader(current) || p.isHRule(current) { + p.renderParagraph(out, data[:i]) + return i + } + + // if there's a fenced code block, paragraph is over + if p.flags&EXTENSION_FENCED_CODE != 0 { + if p.fencedCodeBlock(out, current, false) > 0 { + p.renderParagraph(out, data[:i]) + return i + } + } + + // if there's a definition list item, prev line is a definition term + if p.flags&EXTENSION_DEFINITION_LISTS != 0 { + if p.dliPrefix(current) != 0 { + return p.list(out, data[prev:], LIST_TYPE_DEFINITION) + } + } + + // if there's a list after this, paragraph is over + if p.flags&EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK != 0 { + if p.uliPrefix(current) != 0 || + p.oliPrefix(current) != 0 || + p.quotePrefix(current) != 0 || + p.codePrefix(current) != 0 { + p.renderParagraph(out, data[:i]) + return i + } + } + + // otherwise, scan to the beginning of the next line + for data[i] != '\n' { + i++ + } + i++ + } + + p.renderParagraph(out, data[:i]) + return i +} + +// SanitizedAnchorName returns a sanitized anchor name for the given text. +// +// It implements the algorithm specified in the package comment. +func SanitizedAnchorName(text string) string { + var anchorName []rune + futureDash := false + for _, r := range text { + switch { + case unicode.IsLetter(r) || unicode.IsNumber(r): + if futureDash && len(anchorName) > 0 { + anchorName = append(anchorName, '-') + } + futureDash = false + anchorName = append(anchorName, unicode.ToLower(r)) + default: + futureDash = true + } + } + return string(anchorName) +} diff --git a/vendor/github.com/russross/blackfriday/doc.go b/vendor/github.com/russross/blackfriday/doc.go new file mode 100644 index 000000000..9656c42a1 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/doc.go @@ -0,0 +1,32 @@ +// Package blackfriday is a Markdown processor. +// +// It translates plain text with simple formatting rules into HTML or LaTeX. +// +// Sanitized Anchor Names +// +// Blackfriday includes an algorithm for creating sanitized anchor names +// corresponding to a given input text. This algorithm is used to create +// anchors for headings when EXTENSION_AUTO_HEADER_IDS is enabled. The +// algorithm is specified below, so that other packages can create +// compatible anchor names and links to those anchors. +// +// The algorithm iterates over the input text, interpreted as UTF-8, +// one Unicode code point (rune) at a time. All runes that are letters (category L) +// or numbers (category N) are considered valid characters. They are mapped to +// lower case, and included in the output. All other runes are considered +// invalid characters. Invalid characters that preceed the first valid character, +// as well as invalid character that follow the last valid character +// are dropped completely. All other sequences of invalid characters +// between two valid characters are replaced with a single dash character '-'. +// +// SanitizedAnchorName exposes this functionality, and can be used to +// create compatible links to the anchor names generated by blackfriday. +// This algorithm is also implemented in a small standalone package at +// github.com/shurcooL/sanitized_anchor_name. It can be useful for clients +// that want a small package and don't need full functionality of blackfriday. +package blackfriday + +// NOTE: Keep Sanitized Anchor Name algorithm in sync with package +// github.com/shurcooL/sanitized_anchor_name. +// Otherwise, users of sanitized_anchor_name will get anchor names +// that are incompatible with those generated by blackfriday. diff --git a/vendor/github.com/russross/blackfriday/go.mod b/vendor/github.com/russross/blackfriday/go.mod new file mode 100644 index 000000000..b05561a06 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/go.mod @@ -0,0 +1 @@ +module github.com/russross/blackfriday diff --git a/vendor/github.com/russross/blackfriday/html.go b/vendor/github.com/russross/blackfriday/html.go new file mode 100644 index 000000000..e0a6c69c9 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/html.go @@ -0,0 +1,938 @@ +// +// Blackfriday Markdown Processor +// Available at http://github.com/russross/blackfriday +// +// Copyright © 2011 Russ Ross . +// Distributed under the Simplified BSD License. +// See README.md for details. +// + +// +// +// HTML rendering backend +// +// + +package blackfriday + +import ( + "bytes" + "fmt" + "regexp" + "strconv" + "strings" +) + +// Html renderer configuration options. +const ( + HTML_SKIP_HTML = 1 << iota // skip preformatted HTML blocks + HTML_SKIP_STYLE // skip embedded