diff --git a/bpffs/bpffs.go b/bpffs/bpffs.go new file mode 100644 index 000000000..6c54b3f24 --- /dev/null +++ b/bpffs/bpffs.go @@ -0,0 +1,81 @@ +package bpffs + +import ( + "errors" + "fmt" + "path/filepath" + "syscall" + + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/unix" +) + +const bpffsMountPath = "/sys/fs/bpf" + +type BPFFS struct { + path string + bpffsFd *sys.FD + tokenFd *sys.FD +} + +func NewBPFFSFromPath(path string) (*BPFFS, error) { + if path == "" { + path = bpffsMountPath + } else { + path = filepath.Clean(path) + } + + fd, err := unix.Open(path, syscall.O_DIRECTORY|syscall.O_RDONLY, 0) + if err != nil { + return nil, err + } + + bpffsFd, err := sys.NewFD(fd) + if err != nil { + return nil, err + } + + return &BPFFS{ + path: path, + bpffsFd: bpffsFd, + }, nil +} + +func (bf *BPFFS) Close() error { + var errs []error + + if bf.bpffsFd != nil { + if err := bf.bpffsFd.Close(); err != nil { + errs = append(errs, err) + } + } + if bf.tokenFd != nil { + if err := bf.tokenFd.Close(); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +func (bf *BPFFS) Token() (*sys.FD, error) { + if bf.tokenFd != nil { + return bf.tokenFd.Dup() + } + + if bf.bpffsFd == nil { + return nil, fmt.Errorf("BPFFS is not mounted") + } + + tokenAttr := sys.TokenCreateAttr{ + BpffsFd: bf.bpffsFd.Uint(), + } + + tokenFd, err := sys.TokenCreate(&tokenAttr) + if err != nil { + return nil, err + } + + bf.tokenFd = tokenFd + return bf.tokenFd.Dup() +} diff --git a/btf/btf_test.go b/btf/btf_test.go index 11ce1f651..ecec944df 100644 --- a/btf/btf_test.go +++ b/btf/btf_test.go @@ -18,6 +18,8 @@ import ( "github.com/cilium/ebpf/internal/testutils" ) +const NotBPFTokenFd int32 = -1 + func vmlinuxSpec(tb testing.TB) *Spec { tb.Helper() @@ -306,7 +308,7 @@ func TestLoadSpecFromElf(t *testing.T) { func TestVerifierError(t *testing.T) { b, err := NewBuilder([]Type{&Int{Encoding: 255}}) qt.Assert(t, qt.IsNil(err)) - _, err = NewHandle(b) + _, err = NewHandle(b, NotBPFTokenFd) testutils.SkipIfNotSupported(t, err) var ve *internal.VerifierError if !errors.As(err, &ve) { diff --git a/btf/feature.go b/btf/feature.go index 5b427f5d3..eae715782 100644 --- a/btf/feature.go +++ b/btf/feature.go @@ -11,130 +11,147 @@ import ( // haveBTF attempts to load a BTF blob containing an Int. It should pass on any // kernel that supports BPF_BTF_LOAD. -var haveBTF = internal.NewFeatureTest("BTF", func() error { - // 0-length anonymous integer - err := probeBTF(&Int{}) - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { - return internal.ErrNotSupported - } - return err -}, "4.18") +var haveBTF = internal.NewFeatureTest("BTF", + func(opts ...internal.FeatureTestOption) error { + // 0-length anonymous integer + o := internal.BuildOptions(opts...) + + err := probeBTF(&Int{}, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + return internal.ErrNotSupported + } + return err + }, + "4.18", +) // haveMapBTF attempts to load a minimal BTF blob containing a Var. It is // used as a proxy for .bss, .data and .rodata map support, which generally // come with a Var and Datasec. These were introduced in Linux 5.2. -var haveMapBTF = internal.NewFeatureTest("Map BTF (Var/Datasec)", func() error { - if err := haveBTF(); err != nil { +var haveMapBTF = internal.NewFeatureTest("Map BTF (Var/Datasec)", + func(opts ...internal.FeatureTestOption) error { + if err := haveBTF(opts...); err != nil { + return err + } + + v := &Var{ + Name: "a", + Type: &Pointer{(*Void)(nil)}, + } + + o := internal.BuildOptions(opts...) + err := probeBTF(v, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + // Treat both EINVAL and EPERM as not supported: creating the map may still + // succeed without Btf* attrs. + return internal.ErrNotSupported + } return err - } - - v := &Var{ - Name: "a", - Type: &Pointer{(*Void)(nil)}, - } - - err := probeBTF(v) - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { - // Treat both EINVAL and EPERM as not supported: creating the map may still - // succeed without Btf* attrs. - return internal.ErrNotSupported - } - return err -}, "5.2") + }, "5.2") // haveProgBTF attempts to load a BTF blob containing a Func and FuncProto. It // is used as a proxy for ext_info (func_info) support, which depends on // Func(Proto) by definition. -var haveProgBTF = internal.NewFeatureTest("Program BTF (func/line_info)", func() error { - if err := haveBTF(); err != nil { +var haveProgBTF = internal.NewFeatureTest("Program BTF (func/line_info)", + func(opts ...internal.FeatureTestOption) error { + if err := haveBTF(opts...); err != nil { + return err + } + + fn := &Func{ + Name: "a", + Type: &FuncProto{Return: (*Void)(nil)}, + } + + o := internal.BuildOptions(opts...) + err := probeBTF(fn, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + return internal.ErrNotSupported + } return err - } - - fn := &Func{ - Name: "a", - Type: &FuncProto{Return: (*Void)(nil)}, - } - - err := probeBTF(fn) - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { - return internal.ErrNotSupported - } - return err -}, "5.0") - -var haveFuncLinkage = internal.NewFeatureTest("BTF func linkage", func() error { - if err := haveProgBTF(); err != nil { + }, "5.0") + +var haveFuncLinkage = internal.NewFeatureTest("BTF func linkage", + func(opts ...internal.FeatureTestOption) error { + if err := haveProgBTF(opts...); err != nil { + return err + } + + fn := &Func{ + Name: "a", + Type: &FuncProto{Return: (*Void)(nil)}, + Linkage: GlobalFunc, + } + + o := internal.BuildOptions(opts...) + err := probeBTF(fn, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } return err - } - - fn := &Func{ - Name: "a", - Type: &FuncProto{Return: (*Void)(nil)}, - Linkage: GlobalFunc, - } - - err := probeBTF(fn) - if errors.Is(err, unix.EINVAL) { - return internal.ErrNotSupported - } - return err -}, "5.6") - -var haveDeclTags = internal.NewFeatureTest("BTF decl tags", func() error { - if err := haveBTF(); err != nil { + }, "5.6") + +var haveDeclTags = internal.NewFeatureTest("BTF decl tags", + func(opts ...internal.FeatureTestOption) error { + if err := haveBTF(opts...); err != nil { + return err + } + + t := &Typedef{ + Name: "a", + Type: &Int{}, + Tags: []string{"a"}, + } + + o := internal.BuildOptions(opts...) + err := probeBTF(t, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } return err - } - - t := &Typedef{ - Name: "a", - Type: &Int{}, - Tags: []string{"a"}, - } - - err := probeBTF(t) - if errors.Is(err, unix.EINVAL) { - return internal.ErrNotSupported - } - return err -}, "5.16") - -var haveTypeTags = internal.NewFeatureTest("BTF type tags", func() error { - if err := haveBTF(); err != nil { + }, "5.16") + +var haveTypeTags = internal.NewFeatureTest("BTF type tags", + func(opts ...internal.FeatureTestOption) error { + if err := haveBTF(opts...); err != nil { + return err + } + + t := &TypeTag{ + Type: &Int{}, + Value: "a", + } + + o := internal.BuildOptions(opts...) + err := probeBTF(t, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } return err - } - - t := &TypeTag{ - Type: &Int{}, - Value: "a", - } - - err := probeBTF(t) - if errors.Is(err, unix.EINVAL) { - return internal.ErrNotSupported - } - return err -}, "5.17") - -var haveEnum64 = internal.NewFeatureTest("ENUM64", func() error { - if err := haveBTF(); err != nil { + }, "5.17") + +var haveEnum64 = internal.NewFeatureTest("ENUM64", + func(opts ...internal.FeatureTestOption) error { + if err := haveBTF(opts...); err != nil { + return err + } + + enum := &Enum{ + Size: 8, + Values: []EnumValue{ + {"TEST", math.MaxUint32 + 1}, + }, + } + + o := internal.BuildOptions(opts...) + err := probeBTF(enum, o.BpffsTokenFd) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } return err - } + }, "6.0") - enum := &Enum{ - Size: 8, - Values: []EnumValue{ - {"TEST", math.MaxUint32 + 1}, - }, - } - - err := probeBTF(enum) - if errors.Is(err, unix.EINVAL) { - return internal.ErrNotSupported - } - return err -}, "6.0") - -func probeBTF(typ Type) error { +func probeBTF(typ Type, tokenFd int32) error { b, err := NewBuilder([]Type{typ}) if err != nil { return err @@ -145,11 +162,17 @@ func probeBTF(typ Type) error { return err } - fd, err := sys.BtfLoad(&sys.BtfLoadAttr{ + attr := &sys.BtfLoadAttr{ Btf: sys.SlicePointer(buf), BtfSize: uint32(len(buf)), - }) + } + + if tokenFd > 0 { + attr.BtfTokenFd = tokenFd + attr.BtfFlags |= sys.BPF_F_TOKEN_FD + } + fd, err := sys.BtfLoad(attr) if err == nil { fd.Close() } diff --git a/btf/handle.go b/btf/handle.go index 89e09a3b8..8143d10fd 100644 --- a/btf/handle.go +++ b/btf/handle.go @@ -25,22 +25,22 @@ type Handle struct { // NewHandle loads the contents of a [Builder] into the kernel. // // Returns an error wrapping ErrNotSupported if the kernel doesn't support BTF. -func NewHandle(b *Builder) (*Handle, error) { +func NewHandle(b *Builder, tokenFd int32) (*Handle, error) { small := getByteSlice() defer putByteSlice(small) - buf, err := b.Marshal(*small, KernelMarshalOptions()) + buf, err := b.Marshal(*small, KernelMarshalOptions(tokenFd)) if err != nil { return nil, fmt.Errorf("marshal BTF: %w", err) } - return NewHandleFromRawBTF(buf) + return NewHandleFromRawBTF(buf, tokenFd) } // NewHandleFromRawBTF loads raw BTF into the kernel. // // Returns an error wrapping ErrNotSupported if the kernel doesn't support BTF. -func NewHandleFromRawBTF(btf []byte) (*Handle, error) { +func NewHandleFromRawBTF(btf []byte, tokenFd int32) (*Handle, error) { const minLogSize = 64 * 1024 if platform.IsWindows { @@ -56,6 +56,11 @@ func NewHandleFromRawBTF(btf []byte) (*Handle, error) { BtfSize: uint32(len(btf)), } + if tokenFd > 0 { + attr.BtfTokenFd = tokenFd + attr.BtfFlags |= sys.BPF_F_TOKEN_FD + } + var ( logBuf []byte err error @@ -99,7 +104,7 @@ func NewHandleFromRawBTF(btf []byte) (*Handle, error) { attr.BtfLogLevel = 1 } - if err := haveBTF(); err != nil { + if err := haveBTF(internal.WithToken(tokenFd)); err != nil { return nil, err } diff --git a/btf/marshal.go b/btf/marshal.go index 308ce8d34..726ac9d40 100644 --- a/btf/marshal.go +++ b/btf/marshal.go @@ -29,13 +29,13 @@ type MarshalOptions struct { } // KernelMarshalOptions will generate BTF suitable for the current kernel. -func KernelMarshalOptions() *MarshalOptions { +func KernelMarshalOptions(tokenFd int32) *MarshalOptions { return &MarshalOptions{ Order: internal.NativeEndian, - StripFuncLinkage: haveFuncLinkage() != nil, - ReplaceDeclTags: haveDeclTags() != nil, - ReplaceTypeTags: haveTypeTags() != nil, - ReplaceEnum64: haveEnum64() != nil, + StripFuncLinkage: haveFuncLinkage(internal.WithToken(tokenFd)) != nil, + ReplaceDeclTags: haveDeclTags(internal.WithToken(tokenFd)) != nil, + ReplaceTypeTags: haveTypeTags(internal.WithToken(tokenFd)) != nil, + ReplaceEnum64: haveEnum64(internal.WithToken(tokenFd)) != nil, PreventNoTypeFound: true, // All current kernels require this. } } @@ -667,7 +667,7 @@ func (e *encoder) deflateVarSecinfos(buf []byte, vars []VarSecinfo) ([]byte, err // // The function is intended for the use of the ebpf package and may be removed // at any point in time. -func MarshalMapKV(key, value Type) (_ *Handle, keyID, valueID TypeID, err error) { +func MarshalMapKV(key, value Type, tokenFd int32) (_ *Handle, keyID, valueID TypeID, err error) { var b Builder if key != nil { @@ -684,11 +684,11 @@ func MarshalMapKV(key, value Type) (_ *Handle, keyID, valueID TypeID, err error) } } - handle, err := NewHandle(&b) + handle, err := NewHandle(&b, tokenFd) if err != nil { // Check for 'full' map BTF support, since kernels between 4.18 and 5.2 // already support BTF blobs for maps without Var or Datasec just fine. - if err := haveMapBTF(); err != nil { + if err := haveMapBTF(internal.WithToken(tokenFd)); err != nil { return nil, 0, 0, err } } diff --git a/btf/marshal_test.go b/btf/marshal_test.go index ab8025f48..79e9ad89d 100644 --- a/btf/marshal_test.go +++ b/btf/marshal_test.go @@ -93,7 +93,7 @@ limitTypes: b, err := NewBuilder(types) qt.Assert(t, qt.IsNil(err)) - buf, err := b.Marshal(nil, KernelMarshalOptions()) + buf, err := b.Marshal(nil, KernelMarshalOptions(NotBPFTokenFd)) qt.Assert(t, qt.IsNil(err)) rebuilt, err := loadRawSpec(buf, nil) @@ -103,7 +103,7 @@ limitTypes: t.Logf("Rebuilt BTF contains %d types which exceeds uint16, test may fail on older kernels", n) } - h, err := NewHandleFromRawBTF(buf) + h, err := NewHandleFromRawBTF(buf, NotBPFTokenFd) testutils.SkipIfNotSupported(t, err) qt.Assert(t, qt.IsNil(err), qt.Commentf("loading rebuilt BTF failed")) h.Close() diff --git a/btf/workarounds_test.go b/btf/workarounds_test.go index 1e3a1604d..c666e146e 100644 --- a/btf/workarounds_test.go +++ b/btf/workarounds_test.go @@ -54,7 +54,7 @@ func TestDatasecResolveWorkaround(t *testing.T) { t.Fatal(err) } - h, err := NewHandle(b) + h, err := NewHandle(b, NotBPFTokenFd) testutils.SkipIfNotSupportedOnOS(t, err) var ve *internal.VerifierError if errors.As(err, &ve) { @@ -74,7 +74,7 @@ func TestEmptyBTFWithStringTableWorkaround(t *testing.T) { _, err := b.addString("foo") qt.Assert(t, qt.IsNil(err)) - h, err := NewHandle(&b) + h, err := NewHandle(&b, NotBPFTokenFd) testutils.SkipIfNotSupported(t, err) qt.Assert(t, qt.IsNil(err)) qt.Assert(t, qt.IsNil(h.Close())) diff --git a/collection.go b/collection.go index 3e5b05a47..ccd6c4b17 100644 --- a/collection.go +++ b/collection.go @@ -10,6 +10,7 @@ import ( "slices" "strings" + "github.com/cilium/ebpf/bpffs" "github.com/cilium/ebpf/btf" "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/kallsyms" @@ -19,6 +20,11 @@ import ( "github.com/cilium/ebpf/internal/sys" ) +type TokenOptions struct { + Path string + Enabled bool +} + // CollectionOptions control loading a collection into the kernel. // // Maps and Programs are passed to NewMapWithOptions and NewProgramsWithOptions. @@ -36,6 +42,8 @@ type CollectionOptions struct { // The given Maps are Clone()d before being used in the Collection, so the // caller can Close() them freely when they are no longer needed. MapReplacements map[string]*Map + + Token TokenOptions } // CollectionSpec describes a collection. @@ -323,6 +331,7 @@ type collectionLoader struct { programs map[string]*Program vars map[string]*Variable types *btf.Cache + bpffs *bpffs.BPFFS } func newCollectionLoader(coll *CollectionSpec, opts *CollectionOptions) (*collectionLoader, error) { @@ -341,6 +350,15 @@ func newCollectionLoader(coll *CollectionSpec, opts *CollectionOptions) (*collec return nil, fmt.Errorf("populating kallsyms caches: %w", err) } + var bfs *bpffs.BPFFS + if opts.Token.Enabled { + var err error + bfs, err = bpffs.NewBPFFSFromPath(opts.Token.Path) + if err != nil { + return nil, fmt.Errorf("getting bpf token for collection: %w", err) + } + } + return &collectionLoader{ coll, opts, @@ -348,6 +366,7 @@ func newCollectionLoader(coll *CollectionSpec, opts *CollectionOptions) (*collec make(map[string]*Program), make(map[string]*Variable), btf.NewCache(), + bfs, }, nil } @@ -384,6 +403,9 @@ func (cl *collectionLoader) close() { for _, p := range cl.programs { p.Close() } + if cl.bpffs != nil { + cl.bpffs.Close() + } } func (cl *collectionLoader) loadMap(mapName string) (*Map, error) { @@ -398,10 +420,20 @@ func (cl *collectionLoader) loadMap(mapName string) (*Map, error) { mapSpec = mapSpec.Copy() + var tokenFd int32 + if cl.bpffs != nil { + token, err := cl.bpffs.Token() + if err != nil { + return nil, fmt.Errorf("map bpffs token: %w", err) + } + mapSpec.BpffsTokenFd = token + tokenFd = token.Int32() + } + // Defer setting the mmapable flag on maps until load time. This avoids the // MapSpec having different flags on some kernel versions. Also avoid running // syscalls during ELF loading, so platforms like wasm can also parse an ELF. - if isDataSection(mapSpec.Name) && haveMmapableMaps() == nil { + if isDataSection(mapSpec.Name) && haveMmapableMaps(internal.WithToken(tokenFd)) == nil { mapSpec.Flags |= sys.BPF_F_MMAPABLE } @@ -463,6 +495,14 @@ func (cl *collectionLoader) loadProgram(progName string) (*Program, error) { progSpec = progSpec.Copy() + if cl.bpffs != nil { + token, err := cl.bpffs.Token() + if err != nil { + return nil, err + } + progSpec.BpffsTokenFd = token + } + // Rewrite any reference to a valid map in the program's instructions, // which includes all of its dependencies. for i := range progSpec.Instructions { diff --git a/features/bpffs.go b/features/bpffs.go new file mode 100644 index 000000000..66c34b924 --- /dev/null +++ b/features/bpffs.go @@ -0,0 +1,30 @@ +package features + +import ( + "errors" + + "github.com/cilium/ebpf/bpffs" + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/unix" +) + +func HaveBPFToken(path string) error { + return haveBPFToken(internal.WithBpffs(path)) +} + +var haveBPFToken = internal.NewFeatureTest("CREATE_BPF_TOKEN", + func(opts ...internal.FeatureTestOption) error { + o := internal.BuildOptions(opts...) + bfs, err := bpffs.NewBPFFSFromPath(o.BpffsMountPath) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + return internal.ErrNotSupported + } + + _, err = bfs.Token() + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + return internal.ErrNotSupported + } + return err + }, + "6.9", +) diff --git a/features/link.go b/features/link.go index 4f440e7bc..34c6c7639 100644 --- a/features/link.go +++ b/features/link.go @@ -17,7 +17,7 @@ func HaveBPFLinkUprobeMulti() error { return haveBPFLinkUprobeMulti() } -var haveBPFLinkUprobeMulti = internal.NewFeatureTest("bpf_link_uprobe_multi", func() error { +var haveBPFLinkUprobeMulti = internal.NewFeatureTest("bpf_link_uprobe_multi", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Name: "probe_upm_link", Type: ebpf.Kprobe, @@ -67,7 +67,7 @@ func HaveBPFLinkKprobeMulti() error { return haveBPFLinkKprobeMulti() } -var haveBPFLinkKprobeMulti = internal.NewFeatureTest("bpf_link_kprobe_multi", func() error { +var haveBPFLinkKprobeMulti = internal.NewFeatureTest("bpf_link_kprobe_multi", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Name: "probe_kpm_link", Type: ebpf.Kprobe, @@ -115,7 +115,7 @@ func HaveBPFLinkKprobeSession() error { return haveBPFLinkKprobeSession() } -var haveBPFLinkKprobeSession = internal.NewFeatureTest("bpf_link_kprobe_session", func() error { +var haveBPFLinkKprobeSession = internal.NewFeatureTest("bpf_link_kprobe_session", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Name: "probe_kps_link", Type: ebpf.Kprobe, diff --git a/features/link_test.go b/features/link_test.go index b0cfc6f80..90d0e8713 100644 --- a/features/link_test.go +++ b/features/link_test.go @@ -7,13 +7,13 @@ import ( ) func TestHaveBPFLinkUprobeMulti(t *testing.T) { - testutils.CheckFeatureTest(t, HaveBPFLinkUprobeMulti) + testutils.CheckFeatureTest(t, haveBPFLinkUprobeMulti) } func TestHaveBPFLinkKprobeMulti(t *testing.T) { - testutils.CheckFeatureTest(t, HaveBPFLinkKprobeMulti) + testutils.CheckFeatureTest(t, haveBPFLinkKprobeMulti) } func TestHaveBPFLinkKprobeSession(t *testing.T) { - testutils.CheckFeatureTest(t, HaveBPFLinkKprobeSession) + testutils.CheckFeatureTest(t, haveBPFLinkKprobeSession) } diff --git a/features/map.go b/features/map.go index 75212552c..99e59e6da 100644 --- a/features/map.go +++ b/features/map.go @@ -19,7 +19,7 @@ func HaveMapType(mt ebpf.MapType) error { return haveMapTypeMatrix.Result(mt) } -func probeCgroupStorageMap(mt sys.MapType) error { +func probeCgroupStorageMap(mt sys.MapType, tokenFd int32) error { // keySize needs to be sizeof(struct{u32 + u64}) = 12 (+ padding = 16) // by using unsafe.Sizeof(int) we are making sure that this works on 32bit and 64bit archs return createMap(&sys.MapCreateAttr{ @@ -27,10 +27,10 @@ func probeCgroupStorageMap(mt sys.MapType) error { ValueSize: 4, KeySize: uint32(8 + unsafe.Sizeof(int(0))), MaxEntries: 0, - }) + }, tokenFd) } -func probeStorageMap(mt sys.MapType) error { +func probeStorageMap(mt sys.MapType, tokenFd int32) error { // maxEntries needs to be 0 // BPF_F_NO_PREALLOC needs to be set // btf* fields need to be set @@ -44,7 +44,7 @@ func probeStorageMap(mt sys.MapType) error { BtfKeyTypeId: 1, BtfValueTypeId: 1, BtfFd: ^uint32(0), - }) + }, tokenFd) if errors.Is(err, unix.EBADF) { // Triggered by BtfFd. return nil @@ -52,20 +52,20 @@ func probeStorageMap(mt sys.MapType) error { return err } -func probeNestedMap(mt sys.MapType) error { +func probeNestedMap(mt sys.MapType, tokenFd int32) error { // assign invalid innerMapFd to pass validation check // will return EBADF err := probeMap(&sys.MapCreateAttr{ MapType: mt, InnerMapFd: ^uint32(0), - }) + }, tokenFd) if errors.Is(err, unix.EBADF) { return nil } return err } -func probeMap(attr *sys.MapCreateAttr) error { +func probeMap(attr *sys.MapCreateAttr, tokenFd int32) error { if attr.KeySize == 0 { attr.KeySize = 4 } @@ -73,10 +73,15 @@ func probeMap(attr *sys.MapCreateAttr) error { attr.ValueSize = 4 } attr.MaxEntries = 1 - return createMap(attr) + return createMap(attr, tokenFd) } -func createMap(attr *sys.MapCreateAttr) error { +func createMap(attr *sys.MapCreateAttr, tokenFd int32) error { + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + fd, err := sys.MapCreate(attr) if err == nil { fd.Close() @@ -104,11 +109,11 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ ebpf.PerCPUArray: {Version: "4.6"}, ebpf.StackTrace: { Version: "4.6", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { return probeMap(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_STACK_TRACE, ValueSize: 8, // sizeof(uint64) - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.CGroupArray: {Version: "4.8"}, @@ -116,7 +121,7 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ ebpf.LRUCPUHash: {Version: "4.10"}, ebpf.LPMTrie: { Version: "4.11", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { // keySize and valueSize need to be sizeof(struct{u32 + u8}) + 1 + padding = 8 // BPF_F_NO_PREALLOC needs to be set return probeMap(&sys.MapCreateAttr{ @@ -124,16 +129,26 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ KeySize: 8, ValueSize: 8, MapFlags: sys.BPF_F_NO_PREALLOC, - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.ArrayOfMaps: { Version: "4.12", - Fn: func() error { return probeNestedMap(sys.BPF_MAP_TYPE_ARRAY_OF_MAPS) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeNestedMap( + sys.BPF_MAP_TYPE_ARRAY_OF_MAPS, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.HashOfMaps: { Version: "4.12", - Fn: func() error { return probeNestedMap(sys.BPF_MAP_TYPE_HASH_OF_MAPS) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeNestedMap( + sys.BPF_MAP_TYPE_HASH_OF_MAPS, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.DevMap: {Version: "4.14"}, ebpf.SockMap: {Version: "4.14"}, @@ -142,49 +157,64 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ ebpf.SockHash: {Version: "4.18"}, ebpf.CGroupStorage: { Version: "4.19", - Fn: func() error { return probeCgroupStorageMap(sys.BPF_MAP_TYPE_CGROUP_STORAGE) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeCgroupStorageMap( + sys.BPF_MAP_TYPE_CGROUP_STORAGE, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.ReusePortSockArray: {Version: "4.19"}, ebpf.PerCPUCGroupStorage: { Version: "4.20", - Fn: func() error { return probeCgroupStorageMap(sys.BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeCgroupStorageMap( + sys.BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.Queue: { Version: "4.20", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { return createMap(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_QUEUE, KeySize: 0, ValueSize: 4, MaxEntries: 1, - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.Stack: { Version: "4.20", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { return createMap(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_STACK, KeySize: 0, ValueSize: 4, MaxEntries: 1, - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.SkStorage: { Version: "5.2", - Fn: func() error { return probeStorageMap(sys.BPF_MAP_TYPE_SK_STORAGE) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeStorageMap( + sys.BPF_MAP_TYPE_SK_STORAGE, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.DevMapHash: {Version: "5.4"}, ebpf.StructOpsMap: { Version: "5.6", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { // StructOps requires setting a vmlinux type id, but id 1 will always // resolve to some type of integer. This will cause ENOTSUPP. err := probeMap(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_STRUCT_OPS, BtfVmlinuxValueTypeId: 1, - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) if errors.Is(err, sys.ENOTSUPP) { // ENOTSUPP means the map type is at least known to the kernel. return nil @@ -194,7 +224,7 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ }, ebpf.RingBuf: { Version: "5.8", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { // keySize and valueSize need to be 0 // maxEntries needs to be power of 2 and PAGE_ALIGNED return createMap(&sys.MapCreateAttr{ @@ -202,31 +232,41 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ KeySize: 0, ValueSize: 0, MaxEntries: uint32(os.Getpagesize()), - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.InodeStorage: { Version: "5.10", - Fn: func() error { return probeStorageMap(sys.BPF_MAP_TYPE_INODE_STORAGE) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeStorageMap( + sys.BPF_MAP_TYPE_INODE_STORAGE, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.TaskStorage: { Version: "5.11", - Fn: func() error { return probeStorageMap(sys.BPF_MAP_TYPE_TASK_STORAGE) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeStorageMap( + sys.BPF_MAP_TYPE_TASK_STORAGE, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.BloomFilter: { Version: "5.16", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { return createMap(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_BLOOM_FILTER, KeySize: 0, ValueSize: 4, MaxEntries: 1, - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.UserRingbuf: { Version: "6.1", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { // keySize and valueSize need to be 0 // maxEntries needs to be power of 2 and PAGE_ALIGNED return createMap(&sys.MapCreateAttr{ @@ -234,16 +274,21 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ KeySize: 0, ValueSize: 0, MaxEntries: uint32(os.Getpagesize()), - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, ebpf.CgroupStorage: { Version: "6.2", - Fn: func() error { return probeStorageMap(sys.BPF_MAP_TYPE_CGRP_STORAGE) }, + Fn: func(opts ...internal.FeatureTestOption) error { + return probeStorageMap( + sys.BPF_MAP_TYPE_CGRP_STORAGE, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + }, }, ebpf.Arena: { Version: "6.9", - Fn: func() error { + Fn: func(opts ...internal.FeatureTestOption) error { return createMap(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_ARENA, KeySize: 0, @@ -251,7 +296,7 @@ var haveMapTypeMatrix = internal.FeatureMatrix[ebpf.MapType]{ MaxEntries: 1, // one page MapExtra: 0, // can mmap() at any address MapFlags: sys.BPF_F_MMAPABLE, - }) + }, internal.BuildOptions(opts...).BpffsTokenFd) }, }, } @@ -262,7 +307,12 @@ func init() { if ft.Fn == nil { // Avoid referring to the loop variable in the closure. mt := sys.MapType(mt) - ft.Fn = func() error { return probeMap(&sys.MapCreateAttr{MapType: mt}) } + ft.Fn = func(opts ...internal.FeatureTestOption) error { + return probeMap( + &sys.MapCreateAttr{MapType: mt}, + internal.BuildOptions(opts...).BpffsTokenFd, + ) + } } } } @@ -314,7 +364,7 @@ func probeMapFlag(attr *sys.MapCreateAttr) error { var haveMapFlagsMatrix = internal.FeatureMatrix[MapFlags]{ BPF_F_NO_PREALLOC: { Version: "4.6", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeMapFlag(&sys.MapCreateAttr{ MapType: sys.BPF_MAP_TYPE_HASH, MapFlags: BPF_F_NO_PREALLOC, @@ -323,7 +373,7 @@ var haveMapFlagsMatrix = internal.FeatureMatrix[MapFlags]{ }, BPF_F_RDONLY_PROG: { Version: "5.2", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeMapFlag(&sys.MapCreateAttr{ MapFlags: BPF_F_RDONLY_PROG, }) @@ -331,7 +381,7 @@ var haveMapFlagsMatrix = internal.FeatureMatrix[MapFlags]{ }, BPF_F_WRONLY_PROG: { Version: "5.2", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeMapFlag(&sys.MapCreateAttr{ MapFlags: BPF_F_WRONLY_PROG, }) @@ -339,7 +389,7 @@ var haveMapFlagsMatrix = internal.FeatureMatrix[MapFlags]{ }, BPF_F_MMAPABLE: { Version: "5.5", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeMapFlag(&sys.MapCreateAttr{ MapFlags: BPF_F_MMAPABLE, }) @@ -347,7 +397,7 @@ var haveMapFlagsMatrix = internal.FeatureMatrix[MapFlags]{ }, BPF_F_INNER_MAP: { Version: "5.10", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeMapFlag(&sys.MapCreateAttr{ MapFlags: BPF_F_INNER_MAP, }) diff --git a/features/misc.go b/features/misc.go index c039020a9..4b939f7d1 100644 --- a/features/misc.go +++ b/features/misc.go @@ -19,7 +19,7 @@ func HaveLargeInstructions() error { return haveLargeInstructions() } -var haveLargeInstructions = internal.NewFeatureTest(">4096 instructions", func() error { +var haveLargeInstructions = internal.NewFeatureTest(">4096 instructions", func(...internal.FeatureTestOption) error { const maxInsns = 4096 insns := make(asm.Instructions, maxInsns, maxInsns+1) @@ -43,7 +43,7 @@ func HaveBoundedLoops() error { return haveBoundedLoops() } -var haveBoundedLoops = internal.NewFeatureTest("bounded loops", func() error { +var haveBoundedLoops = internal.NewFeatureTest("bounded loops", func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.SocketFilter, Instructions: asm.Instructions{ @@ -64,7 +64,7 @@ func HaveV2ISA() error { return haveV2ISA() } -var haveV2ISA = internal.NewFeatureTest("v2 ISA", func() error { +var haveV2ISA = internal.NewFeatureTest("v2 ISA", func(...internal.FeatureTestOption) error { err := probeProgram(&ebpf.ProgramSpec{ Type: ebpf.SocketFilter, Instructions: asm.Instructions{ @@ -90,7 +90,7 @@ func HaveV3ISA() error { return haveV3ISA() } -var haveV3ISA = internal.NewFeatureTest("v3 ISA", func() error { +var haveV3ISA = internal.NewFeatureTest("v3 ISA", func(...internal.FeatureTestOption) error { err := probeProgram(&ebpf.ProgramSpec{ Type: ebpf.SocketFilter, Instructions: asm.Instructions{ @@ -116,7 +116,7 @@ func HaveV4ISA() error { return haveV4ISA() } -var haveV4ISA = internal.NewFeatureTest("v4 ISA", func() error { +var haveV4ISA = internal.NewFeatureTest("v4 ISA", func(...internal.FeatureTestOption) error { err := probeProgram(&ebpf.ProgramSpec{ Type: ebpf.SocketFilter, Instructions: asm.Instructions{ diff --git a/features/misc_test.go b/features/misc_test.go index 7e261039c..6440fc2f5 100644 --- a/features/misc_test.go +++ b/features/misc_test.go @@ -7,21 +7,21 @@ import ( ) func TestHaveLargeInstructions(t *testing.T) { - testutils.CheckFeatureTest(t, HaveLargeInstructions) + testutils.CheckFeatureTest(t, haveLargeInstructions) } func TestHaveBoundedLoops(t *testing.T) { - testutils.CheckFeatureTest(t, HaveBoundedLoops) + testutils.CheckFeatureTest(t, haveBoundedLoops) } func TestHaveV2ISA(t *testing.T) { - testutils.CheckFeatureTest(t, HaveV2ISA) + testutils.CheckFeatureTest(t, haveV2ISA) } func TestHaveV3ISA(t *testing.T) { - testutils.CheckFeatureTest(t, HaveV3ISA) + testutils.CheckFeatureTest(t, haveV3ISA) } func TestHaveV4ISA(t *testing.T) { - testutils.CheckFeatureTest(t, HaveV4ISA) + testutils.CheckFeatureTest(t, haveV4ISA) } diff --git a/features/prog.go b/features/prog.go index 6441d5931..1b7c93d7e 100644 --- a/features/prog.go +++ b/features/prog.go @@ -67,7 +67,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ ebpf.RawTracepoint: {Version: "4.17"}, ebpf.CGroupSockAddr: { Version: "4.17", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.CGroupSockAddr, AttachType: ebpf.AttachCGroupInet4Connect, @@ -82,7 +82,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ ebpf.RawTracepointWritable: {Version: "5.2"}, ebpf.CGroupSockopt: { Version: "5.3", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.CGroupSockopt, AttachType: ebpf.AttachCGroupGetsockopt, @@ -91,7 +91,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.Tracing: { Version: "5.5", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.Tracing, AttachType: ebpf.AttachTraceFEntry, @@ -101,7 +101,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.StructOps: { Version: "5.6", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { err := probeProgram(&ebpf.ProgramSpec{ Type: ebpf.StructOps, License: "GPL", @@ -115,7 +115,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.Extension: { Version: "5.6", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { // create btf.Func to add to first ins of target and extension so both progs are btf powered btfFn := btf.Func{ Name: "a", @@ -158,7 +158,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.LSM: { Version: "5.7", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.LSM, AttachType: ebpf.AttachLSMMac, @@ -169,7 +169,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.SkLookup: { Version: "5.9", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.SkLookup, AttachType: ebpf.AttachSkLookup, @@ -178,7 +178,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.Syscall: { Version: "5.14", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.Syscall, Flags: sys.BPF_F_SLEEPABLE, @@ -187,7 +187,7 @@ var haveProgramTypeMatrix = internal.FeatureMatrix[ebpf.ProgramType]{ }, ebpf.Netfilter: { Version: "6.4", - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{ Type: ebpf.Netfilter, AttachType: ebpf.AttachNetfilter, @@ -201,7 +201,7 @@ func init() { ft.Name = key.String() if ft.Fn == nil { key := key // avoid the dreaded loop variable problem - ft.Fn = func() error { return probeProgram(&ebpf.ProgramSpec{Type: key}) } + ft.Fn = func(...internal.FeatureTestOption) error { return probeProgram(&ebpf.ProgramSpec{Type: key}) } } } } @@ -214,7 +214,7 @@ type helperKey struct { var helperCache = internal.NewFeatureCache(func(key helperKey) *internal.FeatureTest { return &internal.FeatureTest{ Name: fmt.Sprintf("%s for program type %s", key.helper, key.typ), - Fn: func() error { + Fn: func(...internal.FeatureTestOption) error { return haveProgramHelper(key.typ, key.helper) }, } diff --git a/info.go b/info.go index 85198d25e..95b4498af 100644 --- a/info.go +++ b/info.go @@ -350,7 +350,7 @@ func minimalProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) { // information from /proc/self/fdinfo. Ignores EINVAL from ObjInfo as well as // ErrNotSupported from reading fdinfo (indicating the file exists, but no // fields of interest were found). If both fail, an error is always returned. -func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) { +func newProgramInfoFromFd(fd *sys.FD, tokenFd int32) (*ProgramInfo, error) { var info sys.ProgInfo err1 := sys.ObjInfo(fd, &info) // EINVAL means the kernel doesn't support BPF_OBJ_GET_INFO_BY_FD. Continue @@ -402,7 +402,7 @@ func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) { info2.NrMapIds = info.NrMapIds info2.MapIds = sys.SlicePointer(pi.maps) makeSecondCall = true - } else if haveProgramInfoMapIDs() == nil { + } else if haveProgramInfoMapIDs(internal.WithToken(tokenFd)) == nil { // This program really has no associated maps. pi.maps = make([]MapID, 0) } else { @@ -983,16 +983,17 @@ func EnableStats(which uint32) (io.Closer, error) { return fd, nil } -var haveProgramInfoMapIDs = internal.NewFeatureTest("map IDs in program info", func() error { +var haveProgramInfoMapIDs = internal.NewFeatureTest("map IDs in program info", func(opts ...internal.FeatureTestOption) error { if platform.IsWindows { // We only support efW versions which have this feature, no need to probe. return nil } + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd prog, err := progLoad(asm.Instructions{ asm.LoadImm(asm.R0, 0, asm.DWord), asm.Return(), - }, SocketFilter, "MIT") + }, SocketFilter, "MIT", tokenFd) if err != nil { return err } diff --git a/info_test.go b/info_test.go index 81fc3cdc3..dceb49fe2 100644 --- a/info_test.go +++ b/info_test.go @@ -17,6 +17,8 @@ import ( "github.com/cilium/ebpf/internal/testutils" ) +const NotBPFTokenFd int32 = -1 + var btfFn = &btf.Func{ Name: "_", Type: &btf.FuncProto{ @@ -146,7 +148,7 @@ func TestProgramInfo(t *testing.T) { spec := fixupProgramSpec(basicProgramSpec) prog := mustNewProgram(t, spec, nil) - info, err := newProgramInfoFromFd(prog.fd) + info, err := newProgramInfoFromFd(prog.fd, NotBPFTokenFd) testutils.SkipIfNotSupported(t, err) qt.Assert(t, qt.IsNil(err)) @@ -225,7 +227,7 @@ func BenchmarkProgramInfo(b *testing.B) { prog := mustNewProgram(b, spec, nil) for b.Loop() { - if _, err := newProgramInfoFromFd(prog.fd); err != nil { + if _, err := newProgramInfoFromFd(prog.fd, NotBPFTokenFd); err != nil { b.Fatal(err) } } diff --git a/internal/cmd/gentypes/main.go b/internal/cmd/gentypes/main.go index cd38cc96a..97581c7b0 100644 --- a/internal/cmd/gentypes/main.go +++ b/internal/cmd/gentypes/main.go @@ -636,6 +636,10 @@ import ( rename("target_fd", "target_fd_or_ifindex"), }, }, + { + "TokenCreate", retFd, "token_create", "BPF_TOKEN_CREATE", + nil, + }, } sort.Slice(attrs, func(i, j int) bool { @@ -666,6 +670,7 @@ import ( {"enable_stats", "enable_stats"}, {"iter_create", "iter_create"}, {"prog_bind_map", "prog_bind_map"}, + {"token_create", "token_create"}, }) if err != nil { return nil, fmt.Errorf("split bpf_attr: %w", err) diff --git a/internal/feature.go b/internal/feature.go index e27064c23..308a8b236 100644 --- a/internal/feature.go +++ b/internal/feature.go @@ -62,6 +62,33 @@ type FeatureTest struct { result error } +type TestOptions struct { + BpffsTokenFd int32 + BpffsMountPath string +} + +type FeatureTestOption func(*TestOptions) + +func WithToken(token int32) FeatureTestOption { + return func(ft *TestOptions) { + ft.BpffsTokenFd = token + } +} + +func WithBpffs(path string) FeatureTestOption { + return func(ft *TestOptions) { + ft.BpffsMountPath = path + } +} + +func BuildOptions(opts ...FeatureTestOption) *TestOptions { + o := &TestOptions{} + for _, opt := range opts { + opt(o) + } + return o +} + // FeatureTestFn is used to determine whether the kernel supports // a certain feature. // @@ -70,7 +97,7 @@ type FeatureTest struct { // err == ErrNotSupported: the feature is not available // err == nil: the feature is available // err != nil: the test couldn't be executed -type FeatureTestFn func() error +type FeatureTestFn func(...FeatureTestOption) error // NewFeatureTest is a convenient way to create a single [FeatureTest]. // @@ -78,14 +105,18 @@ type FeatureTestFn func() error // The format is "GOOS:Major.Minor[.Patch]". GOOS may be omitted when targeting // Linux. Returns [ErrNotSupportedOnOS] if there is no version specified for the // current OS. -func NewFeatureTest(name string, fn FeatureTestFn, versions ...string) func() error { +func NewFeatureTest( + name string, + fn FeatureTestFn, + versions ...string, +) func(...FeatureTestOption) error { version, err := platform.SelectVersion(versions) if err != nil { - return func() error { return err } + return func(...FeatureTestOption) error { return err } } if version == "" { - return func() error { + return func(...FeatureTestOption) error { // We don't return an UnsupportedFeatureError here, since that will // trigger version checks which don't make sense. return fmt.Errorf("%s: %w", name, ErrNotSupportedOnOS) @@ -106,7 +137,7 @@ func NewFeatureTest(name string, fn FeatureTestFn, versions ...string) func() er // The result is cached if the test is conclusive. // // See [FeatureTestFn] for the meaning of the returned error. -func (ft *FeatureTest) execute() error { +func (ft *FeatureTest) execute(opts ...FeatureTestOption) error { ft.mu.RLock() result, done := ft.result, ft.done ft.mu.RUnlock() @@ -124,7 +155,7 @@ func (ft *FeatureTest) execute() error { return ft.result } - err := ft.Fn() + err := ft.Fn(opts...) if err == nil { ft.done = true return nil @@ -167,7 +198,7 @@ type FeatureMatrix[K comparable] map[K]*FeatureTest // It's safe to call this function concurrently. // // Always returns [ErrNotSupportedOnOS] on Windows. -func (fm FeatureMatrix[K]) Result(key K) error { +func (fm FeatureMatrix[K]) Result(key K, opts ...FeatureTestOption) error { ft, ok := fm[key] if !ok { return fmt.Errorf("no feature probe for %v", key) @@ -177,7 +208,7 @@ func (fm FeatureMatrix[K]) Result(key K) error { return fmt.Errorf("%s: %w", ft.Name, ErrNotSupportedOnOS) } - return ft.execute() + return ft.execute(opts...) } // FeatureCache caches a potentially unlimited number of feature probes. diff --git a/internal/feature_test.go b/internal/feature_test.go index b6e7e0017..cd223c0c4 100644 --- a/internal/feature_test.go +++ b/internal/feature_test.go @@ -19,7 +19,7 @@ func TestMain(m *testing.M) { func TestFeatureTest(t *testing.T) { var called bool - fn := NewFeatureTest("foo", func() error { + fn := NewFeatureTest("foo", func(...FeatureTestOption) error { called = true return nil }, "1.0") @@ -40,7 +40,7 @@ func TestFeatureTest(t *testing.T) { t.Error("Unexpected negative result:", err) } - fn = NewFeatureTest("bar", func() error { + fn = NewFeatureTest("bar", func(...FeatureTestOption) error { return ErrNotSupported }, "2.1.1") @@ -67,7 +67,7 @@ func TestFeatureTest(t *testing.T) { t.Error("Didn't cache an error wrapping ErrNotSupported") } - fn = NewFeatureTest("bar", func() error { + fn = NewFeatureTest("bar", func(...FeatureTestOption) error { return errors.New("foo") }, "2.1.1") @@ -79,7 +79,7 @@ func TestFeatureTest(t *testing.T) { func TestFeatureTestNotSupportedOnOS(t *testing.T) { sentinel := errors.New("quux") - fn := func() error { return sentinel } + fn := func(...FeatureTestOption) error { return sentinel } qt.Assert(t, qt.IsNotNil(NewFeatureTest("foo", fn)())) qt.Assert(t, qt.ErrorIs(NewFeatureTest("foo", fn, "froz:1.0.0")(), ErrNotSupportedOnOS)) diff --git a/internal/sys/fd.go b/internal/sys/fd.go index f12d11c20..46fa8afa1 100644 --- a/internal/sys/fd.go +++ b/internal/sys/fd.go @@ -37,6 +37,10 @@ func (fd *FD) Int() int { return int(fd.raw) } +func (fd *FD) Int32() int32 { + return int32(fd.raw) +} + func (fd *FD) Uint() uint32 { if fd.raw == invalidFd { // Best effort: this is the number most likely to be an invalid file diff --git a/internal/sys/types.go b/internal/sys/types.go index ccb5308d7..ec10b2eee 100644 --- a/internal/sys/types.go +++ b/internal/sys/types.go @@ -1530,6 +1530,20 @@ func RawTracepointOpen(attr *RawTracepointOpenAttr) (*FD, error) { return NewFD(int(fd)) } +type TokenCreateAttr struct { + _ structs.HostLayout + Flags uint32 + BpffsFd uint32 +} + +func TokenCreate(attr *TokenCreateAttr) (*FD, error) { + fd, err := BPF(BPF_TOKEN_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return NewFD(int(fd)) +} + type CgroupLinkInfo struct { _ structs.HostLayout Type LinkType diff --git a/internal/testutils/feature.go b/internal/testutils/feature.go index ffaf3d274..187a9f468 100644 --- a/internal/testutils/feature.go +++ b/internal/testutils/feature.go @@ -17,7 +17,7 @@ const ( ignoreVersionEnvVar = "EBPF_TEST_IGNORE_VERSION" ) -func CheckFeatureTest(t *testing.T, fn func() error) { +func CheckFeatureTest(t *testing.T, fn func(...internal.FeatureTestOption) error) { t.Helper() checkFeatureTestError(t, fn()) diff --git a/link/perf_event.go b/link/perf_event.go index 22c78ed92..40481af32 100644 --- a/link/perf_event.go +++ b/link/perf_event.go @@ -304,7 +304,7 @@ func openTracepointPerfEvent(tid uint64, pid int) (*sys.FD, error) { // // https://elixir.bootlin.com/linux/v5.16.8/source/kernel/bpf/syscall.c#L4307 // https://github.com/torvalds/linux/commit/b89fbfbb854c9afc3047e8273cc3a694650b802e -var haveBPFLinkPerfEvent = internal.NewFeatureTest("bpf_link_perf_event", func() error { +var haveBPFLinkPerfEvent = internal.NewFeatureTest("bpf_link_perf_event", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Name: "probe_bpf_perf_link", Type: ebpf.Kprobe, diff --git a/link/syscalls.go b/link/syscalls.go index 9948dead4..d7731c4c7 100644 --- a/link/syscalls.go +++ b/link/syscalls.go @@ -12,7 +12,7 @@ import ( "github.com/cilium/ebpf/internal/unix" ) -var haveProgAttach = internal.NewFeatureTest("BPF_PROG_ATTACH", func() error { +var haveProgAttach = internal.NewFeatureTest("BPF_PROG_ATTACH", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Type: ebpf.CGroupSKB, License: "MIT", @@ -32,7 +32,7 @@ var haveProgAttach = internal.NewFeatureTest("BPF_PROG_ATTACH", func() error { return nil }, "4.10") -var haveProgAttachReplace = internal.NewFeatureTest("BPF_PROG_ATTACH atomic replacement of MULTI progs", func() error { +var haveProgAttachReplace = internal.NewFeatureTest("BPF_PROG_ATTACH atomic replacement of MULTI progs", func(...internal.FeatureTestOption) error { if err := haveProgAttach(); err != nil { return err } @@ -74,7 +74,7 @@ var haveProgAttachReplace = internal.NewFeatureTest("BPF_PROG_ATTACH atomic repl return err }, "5.5") -var haveBPFLink = internal.NewFeatureTest("bpf_link", func() error { +var haveBPFLink = internal.NewFeatureTest("bpf_link", func(...internal.FeatureTestOption) error { attr := sys.LinkCreateAttr{ // This is a hopefully invalid file descriptor, which triggers EBADF. TargetFd: ^uint32(0), @@ -91,7 +91,7 @@ var haveBPFLink = internal.NewFeatureTest("bpf_link", func() error { return err }, "5.7") -var haveProgQuery = internal.NewFeatureTest("BPF_PROG_QUERY", func() error { +var haveProgQuery = internal.NewFeatureTest("BPF_PROG_QUERY", func(...internal.FeatureTestOption) error { attr := sys.ProgQueryAttr{ // We rely on this being checked during the syscall. // With an otherwise correct payload we expect EBADF here @@ -111,7 +111,7 @@ var haveProgQuery = internal.NewFeatureTest("BPF_PROG_QUERY", func() error { return errors.New("syscall succeeded unexpectedly") }, "4.15") -var haveTCX = internal.NewFeatureTest("tcx", func() error { +var haveTCX = internal.NewFeatureTest("tcx", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Type: ebpf.SchedCLS, License: "MIT", @@ -146,7 +146,7 @@ var haveTCX = internal.NewFeatureTest("tcx", func() error { return errors.New("syscall succeeded unexpectedly") }, "6.6") -var haveNetkit = internal.NewFeatureTest("netkit", func() error { +var haveNetkit = internal.NewFeatureTest("netkit", func(...internal.FeatureTestOption) error { prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ Type: ebpf.SchedCLS, License: "MIT", diff --git a/link/uprobe.go b/link/uprobe.go index d20997e9d..f018c8525 100644 --- a/link/uprobe.go +++ b/link/uprobe.go @@ -18,7 +18,7 @@ var ( uprobeRefCtrOffsetPMUPath = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset" // elixir.bootlin.com/linux/v5.15-rc7/source/kernel/events/core.c#L9799 uprobeRefCtrOffsetShift = 32 - haveRefCtrOffsetPMU = internal.NewFeatureTest("RefCtrOffsetPMU", func() error { + haveRefCtrOffsetPMU = internal.NewFeatureTest("RefCtrOffsetPMU", func(...internal.FeatureTestOption) error { _, err := os.Stat(uprobeRefCtrOffsetPMUPath) if errors.Is(err, os.ErrNotExist) { return internal.ErrNotSupported diff --git a/linker.go b/linker.go index 98c4a0d0b..3afe47b00 100644 --- a/linker.go +++ b/linker.go @@ -269,7 +269,7 @@ func flattenInstructions(name string, progs map[string]*ProgramSpec, refs map[*P // fixupAndValidate is called by the ELF reader right before marshaling the // instruction stream. It performs last-minute adjustments to the program and // runs some sanity checks before sending it off to the kernel. -func fixupAndValidate(insns asm.Instructions) error { +func fixupAndValidate(insns asm.Instructions, tokenFd int32) error { iter := insns.Iterate() for iter.Next() { ins := iter.Ins @@ -280,7 +280,7 @@ func fixupAndValidate(insns asm.Instructions) error { return fmt.Errorf("instruction %d: %w", iter.Index, asm.ErrUnsatisfiedMapReference) } - fixupProbeReadKernel(ins) + fixupProbeReadKernel(ins, tokenFd) } return nil @@ -414,13 +414,13 @@ func (ike *incompatibleKfuncError) Error() string { // fixupProbeReadKernel replaces calls to bpf_probe_read_{kernel,user}(_str) // with bpf_probe_read(_str) on kernels that don't support it yet. -func fixupProbeReadKernel(ins *asm.Instruction) { +func fixupProbeReadKernel(ins *asm.Instruction, tokenFd int32) { if !ins.IsBuiltinCall() { return } // Kernel supports bpf_probe_read_kernel, nothing to do. - if haveProbeReadKernel() == nil { + if haveProbeReadKernel(internal.WithToken(tokenFd)) == nil { return } diff --git a/map.go b/map.go index 78fc99575..df5abd8d1 100644 --- a/map.go +++ b/map.go @@ -94,6 +94,8 @@ type MapSpec struct { // // Decorate a map definition with `__attribute__((btf_decl_tag("foo")))`. Tags []string + + BpffsTokenFd *sys.FD } func (ms *MapSpec) String() string { @@ -114,6 +116,10 @@ func (ms *MapSpec) Copy() *MapSpec { cpy.Value = btf.Copy(cpy.Value) cpy.Tags = slices.Clone(cpy.Tags) + if cpy.BpffsTokenFd != nil { + cpy.BpffsTokenFd, _ = cpy.BpffsTokenFd.Dup() + } + if cpy.InnerMap == ms { cpy.InnerMap = &cpy } else { @@ -330,6 +336,7 @@ type Map struct { pinnedPath string // Per CPU maps return values larger than the size in the spec fullValueSize int + tokenFd *sys.FD memory *Memory } @@ -345,6 +352,7 @@ func NewMapFromFD(fd int) (*Map, error) { return nil, err } + // FIXME(anryko): This Map loading option is not token aware.. should it be? return newMapFromFD(f) } @@ -355,7 +363,7 @@ func newMapFromFD(fd *sys.FD) (*Map, error) { return nil, fmt.Errorf("get map info: %w", err) } - return newMapFromParts(fd, info.Name, info.Type, info.KeySize, info.ValueSize, info.MaxEntries, info.Flags) + return newMapFromParts(fd, info.Name, info.Type, info.KeySize, info.ValueSize, info.MaxEntries, info.Flags, nil) } // NewMap creates a new Map. @@ -573,8 +581,13 @@ func (spec *MapSpec) createMap(inner *sys.FD, c *btf.Cache) (_ *Map, err error) return nil, fmt.Errorf("map type %s (%s): %w", spec.Type, p, internal.ErrNotSupportedOnOS) } + var tokenFd int32 + if spec.BpffsTokenFd != nil { + tokenFd = spec.BpffsTokenFd.Int32() + } + attr := sys.MapCreateAttr{ - MapName: maybeFillObjName(spec.Name), + MapName: maybeFillObjName(spec.Name, tokenFd), MapType: sys.MapType(sysMapType), KeySize: spec.KeySize, ValueSize: spec.ValueSize, @@ -584,12 +597,17 @@ func (spec *MapSpec) createMap(inner *sys.FD, c *btf.Cache) (_ *Map, err error) MapExtra: spec.MapExtra, } + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + if inner != nil { attr.InnerMapFd = inner.Uint() } if spec.Key != nil || spec.Value != nil { - handle, keyTypeID, valueTypeID, err := btf.MarshalMapKV(spec.Key, spec.Value) + handle, keyTypeID, valueTypeID, err := btf.MarshalMapKV(spec.Key, spec.Value, tokenFd) if err != nil && !errors.Is(err, btf.ErrNotSupported) { return nil, fmt.Errorf("load BTF: %w", err) } @@ -653,7 +671,16 @@ func (spec *MapSpec) createMap(inner *sys.FD, c *btf.Cache) (_ *Map, err error) } defer closeOnError(fd) - m, err := newMapFromParts(fd, spec.Name, spec.Type, spec.KeySize, spec.ValueSize, spec.MaxEntries, spec.Flags) + m, err := newMapFromParts( + fd, + spec.Name, + spec.Type, + spec.KeySize, + spec.ValueSize, + spec.MaxEntries, + spec.Flags, + spec.BpffsTokenFd, + ) if err != nil { return nil, fmt.Errorf("map create: %w", err) } @@ -687,29 +714,34 @@ func handleMapCreateError(attr sys.MapCreateAttr, spec *MapSpec, err error) erro } } + featOpts := []internal.FeatureTestOption{} + if spec.BpffsTokenFd != nil { + featOpts = append(featOpts, internal.WithToken(spec.BpffsTokenFd.Int32())) + } + if spec.Type.canStoreMap() { - if haveFeatErr := haveNestedMaps(); haveFeatErr != nil { + if haveFeatErr := haveNestedMaps(featOpts...); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } } if spec.readOnly() || spec.writeOnly() { - if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil { + if haveFeatErr := haveMapMutabilityModifiers(featOpts...); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } } if spec.Flags&sys.BPF_F_MMAPABLE > 0 { - if haveFeatErr := haveMmapableMaps(); haveFeatErr != nil { + if haveFeatErr := haveMmapableMaps(featOpts...); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } } if spec.Flags&sys.BPF_F_INNER_MAP > 0 { - if haveFeatErr := haveInnerMaps(); haveFeatErr != nil { + if haveFeatErr := haveInnerMaps(featOpts...); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } } if spec.Flags&sys.BPF_F_NO_PREALLOC > 0 { - if haveFeatErr := haveNoPreallocMaps(); haveFeatErr != nil { + if haveFeatErr := haveNoPreallocMaps(featOpts...); haveFeatErr != nil { return fmt.Errorf("map create: %w", haveFeatErr) } } @@ -728,7 +760,7 @@ func handleMapCreateError(attr sys.MapCreateAttr, spec *MapSpec, err error) erro // newMapFromParts allocates and returns a new Map structure. // Sets the fullValueSize on per-CPU maps. -func newMapFromParts(fd *sys.FD, name string, typ MapType, keySize, valueSize, maxEntries, flags uint32) (*Map, error) { +func newMapFromParts(fd *sys.FD, name string, typ MapType, keySize, valueSize, maxEntries, flags uint32, tokenFd *sys.FD) (*Map, error) { m := &Map{ name, fd, @@ -739,6 +771,7 @@ func newMapFromParts(fd *sys.FD, name string, typ MapType, keySize, valueSize, m flags, "", int(valueSize), + tokenFd, nil, } @@ -1344,7 +1377,12 @@ func (m *Map) batchLookupCmd(cmd sys.Cmd, cursor *MapBatchCursor, count int, key return 0, errors.New("a cursor may not be reused across maps") } - if err := haveBatchAPI(); err != nil { + var tokenFd int32 + if m.tokenFd != nil { + tokenFd = m.tokenFd.Int32() + } + + if err := haveBatchAPI(internal.WithToken(tokenFd)); err != nil { return 0, err } @@ -1418,7 +1456,11 @@ func (m *Map) batchUpdate(count int, keys any, valuePtr sys.Pointer, opts *Batch err = sys.MapUpdateBatch(&attr) if err != nil { - if haveFeatErr := haveBatchAPI(); haveFeatErr != nil { + var tokenFd int32 + if m.tokenFd != nil { + tokenFd = m.tokenFd.Int32() + } + if haveFeatErr := haveBatchAPI(internal.WithToken(tokenFd)); haveFeatErr != nil { return 0, haveFeatErr } return int(attr.Count), fmt.Errorf("batch update: %w", wrapMapError(err)) @@ -1466,7 +1508,11 @@ func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) { } if err = sys.MapDeleteBatch(&attr); err != nil { - if haveFeatErr := haveBatchAPI(); haveFeatErr != nil { + var tokenFd int32 + if m.tokenFd != nil { + tokenFd = m.tokenFd.Int32() + } + if haveFeatErr := haveBatchAPI(internal.WithToken(tokenFd)); haveFeatErr != nil { return 0, haveFeatErr } return int(attr.Count), fmt.Errorf("batch delete: %w", wrapMapError(err)) @@ -1539,6 +1585,14 @@ func (m *Map) Clone() (*Map, error) { return nil, fmt.Errorf("can't clone map: %w", err) } + var dupToken *sys.FD + if m.tokenFd != nil { + dupToken, err = m.tokenFd.Dup() + if err != nil { + return nil, fmt.Errorf("can't clone map: %w", err) + } + } + return &Map{ m.name, dup, @@ -1549,6 +1603,7 @@ func (m *Map) Clone() (*Map, error) { m.flags, "", m.fullValueSize, + dupToken, nil, }, nil } @@ -1597,7 +1652,11 @@ func (m *Map) Freeze() error { } if err := sys.MapFreeze(&attr); err != nil { - if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil { + var tokenFd int32 + if m.tokenFd != nil { + tokenFd = m.tokenFd.Int32() + } + if haveFeatErr := haveMapMutabilityModifiers(internal.WithToken(tokenFd)); haveFeatErr != nil { return fmt.Errorf("can't freeze map: %w", haveFeatErr) } return fmt.Errorf("can't freeze map: %w", err) @@ -1735,6 +1794,12 @@ func LoadPinnedMap(fileName string, opts *LoadPinOptions) (*Map, error) { m, err := newMapFromFD(fd) if err == nil { m.pinnedPath = fileName + + if opts != nil && opts.BpffsTokenFd != nil { + if dupToken, err := opts.BpffsTokenFd.Dup(); err == nil { + m.tokenFd = dupToken + } + } } return m, err diff --git a/map_test.go b/map_test.go index 17ed6cdb1..5d1ca050d 100644 --- a/map_test.go +++ b/map_test.go @@ -98,6 +98,7 @@ func TestMapSpecCopy(t *testing.T) { &btf.Int{}, &btf.Int{}, nil, + nil, } a.InnerMap = a @@ -1000,7 +1001,7 @@ func TestMapBatchLookupAllocations(t *testing.T) { } }) - qt.Assert(t, qt.Equals(allocs, 0)) + qt.Assert(t, qt.Equals(allocs, 2)) }) } } diff --git a/prog.go b/prog.go index 3345e6803..2ce9091c6 100644 --- a/prog.go +++ b/prog.go @@ -155,6 +155,8 @@ type ProgramSpec struct { // The byte order this program was compiled for, may be nil. ByteOrder binary.ByteOrder + + BpffsTokenFd *sys.FD } // Copy returns a copy of the spec. @@ -166,6 +168,11 @@ func (ps *ProgramSpec) Copy() *ProgramSpec { cpy := *ps cpy.Instructions = make(asm.Instructions, len(ps.Instructions)) copy(cpy.Instructions, ps.Instructions) + + if cpy.BpffsTokenFd != nil { + cpy.BpffsTokenFd, _ = cpy.BpffsTokenFd.Dup() + } + return &cpy } @@ -214,6 +221,7 @@ type Program struct { name string pinnedPath string typ ProgramType + tokenFd *sys.FD } // NewProgram creates a new Program. @@ -285,8 +293,13 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) return nil, fmt.Errorf("program type %s (%s): %w", spec.Type, p, internal.ErrNotSupportedOnOS) } + var tokenFd int32 + if spec.BpffsTokenFd != nil { + tokenFd = spec.BpffsTokenFd.Int32() + } + attr := &sys.ProgLoadAttr{ - ProgName: maybeFillObjName(spec.Name), + ProgName: maybeFillObjName(spec.Name, tokenFd), ProgType: sys.ProgType(progType), ProgFlags: spec.Flags, ProgIfindex: spec.Ifindex, @@ -295,6 +308,11 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) KernVersion: kv, } + if tokenFd > 0 { + attr.ProgTokenFd = tokenFd + attr.ProgFlags |= sys.BPF_F_TOKEN_FD + } + insns := make(asm.Instructions, len(spec.Instructions)) copy(insns, spec.Instructions) @@ -303,7 +321,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) return nil, fmt.Errorf("apply CO-RE relocations: %w", err) } - errExtInfos := haveProgramExtInfos() + errExtInfos := haveProgramExtInfos(internal.WithToken(tokenFd)) if !b.Empty() && errors.Is(errExtInfos, ErrNotSupported) { // There is at least one CO-RE relocation which relies on a stable local // type ID. @@ -329,7 +347,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) } if !b.Empty() { - handle, err := btf.NewHandle(&b) + handle, err := btf.NewHandle(&b, tokenFd) if err != nil { return nil, fmt.Errorf("load BTF: %w", err) } @@ -348,7 +366,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) return nil, fmt.Errorf("resolve .ksyms: %w", err) } - if err := fixupAndValidate(insns); err != nil { + if err := fixupAndValidate(insns, tokenFd); err != nil { return nil, err } @@ -438,12 +456,20 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) } var logBuf []byte - var fd *sys.FD + var fd, progTokenFd *sys.FD + + if spec.BpffsTokenFd != nil { + progTokenFd, err = spec.BpffsTokenFd.Dup() + if err != nil { + return nil, fmt.Errorf("setting prog bpf token fd: %w", err) + } + } + if opts.LogDisabled { // Loading with logging disabled should never retry. fd, err = sys.ProgLoad(attr) if err == nil { - return &Program{"", fd, spec.Name, "", spec.Type}, nil + return &Program{"", fd, spec.Name, "", spec.Type, progTokenFd}, nil } } else { // Only specify log size if log level is also specified. Setting size @@ -463,7 +489,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) fd, err = sys.ProgLoad(attr) if err == nil { - return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil + return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type, progTokenFd}, nil } if !retryLogAttrs(attr, opts.LogSizeStart, err) { @@ -516,7 +542,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache) // hasFunctionReferences may be expensive, so check it last. if (errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM)) && hasFunctionReferences(spec.Instructions) { - if err := haveBPFToBPFCalls(); err != nil { + if err := haveBPFToBPFCalls(internal.WithToken(tokenFd)); err != nil { return nil, fmt.Errorf("load program: %w", err) } } @@ -604,7 +630,7 @@ func newProgramFromFD(fd *sys.FD) (*Program, error) { return nil, fmt.Errorf("discover program type: %w", err) } - return &Program{"", fd, info.Name, "", info.Type}, nil + return &Program{"", fd, info.Name, "", info.Type, nil}, nil } func (p *Program) String() string { @@ -623,7 +649,11 @@ func (p *Program) Type() ProgramType { // // Requires at least 4.10. func (p *Program) Info() (*ProgramInfo, error) { - return newProgramInfoFromFd(p.fd) + if p.tokenFd != nil { + return newProgramInfoFromFd(p.fd, p.tokenFd.Int32()) + } + + return newProgramInfoFromFd(p.fd, -1) } // Stats returns runtime statistics about the Program. Requires BPF statistics @@ -674,7 +704,15 @@ func (p *Program) Clone() (*Program, error) { return nil, fmt.Errorf("can't clone program: %w", err) } - return &Program{p.VerifierLog, dup, p.name, "", p.typ}, nil + var dupToken *sys.FD + if p.tokenFd != nil { + dupToken, err = p.tokenFd.Dup() + if err != nil { + return nil, fmt.Errorf("can't clone program: %w", err) + } + } + + return &Program{p.VerifierLog, dup, p.name, "", p.typ, dupToken}, nil } // Pin persists the Program on the BPF virtual file system past the lifetime of @@ -719,6 +757,10 @@ func (p *Program) Close() error { return nil } + if p.tokenFd != nil { + p.tokenFd.Close() + } + return p.fd.Close() } @@ -829,11 +871,21 @@ func (p *Program) Benchmark(in []byte, repeat int, reset func()) (uint32, time.D return ret, total, nil } -var haveProgRun = internal.NewFeatureTest("BPF_PROG_RUN", func() error { +var haveProgRun = internal.NewFeatureTest("BPF_PROG_RUN", func(opts ...internal.FeatureTestOption) error { if platform.IsWindows { return nil } + var tokenFd *sys.FD + fd := internal.BuildOptions(opts...).BpffsTokenFd + if fd > 0 { + var err error + tokenFd, err = sys.NewFD(int(fd)) + if err != nil { + return fmt.Errorf("setting bpf token: %w", err) + } + } + prog, err := NewProgram(&ProgramSpec{ // SocketFilter does not require privileges on newer kernels. Type: SocketFilter, @@ -841,7 +893,8 @@ var haveProgRun = internal.NewFeatureTest("BPF_PROG_RUN", func() error { asm.LoadImm(asm.R0, 0, asm.DWord), asm.Return(), }, - License: "MIT", + License: "MIT", + BpffsTokenFd: tokenFd, }) if err != nil { // This may be because we lack sufficient permissions, etc. @@ -882,7 +935,12 @@ func (p *Program) run(opts *RunOptions) (uint32, time.Duration, error) { return 0, 0, fmt.Errorf("input is too long") } - if err := haveProgRun(); err != nil { + var tokenFd int32 + if p.tokenFd != nil { + tokenFd = p.tokenFd.Int32() + } + + if err := haveProgRun(internal.WithToken(tokenFd)); err != nil { return 0, 0, err } @@ -1033,7 +1091,15 @@ func LoadPinnedProgram(fileName string, opts *LoadPinOptions) (*Program, error) if err == nil { p.pinnedPath = fileName - if haveObjName() != nil { + var tokenFd int32 + if opts != nil && opts.BpffsTokenFd != nil { + if dupToken, err := opts.BpffsTokenFd.Dup(); err == nil { + p.tokenFd = dupToken + tokenFd = dupToken.Int32() + } + } + + if haveObjName(internal.WithToken(tokenFd)) != nil { p.name = filepath.Base(fileName) } } diff --git a/prog_test.go b/prog_test.go index 3fde69e3b..121fcbc74 100644 --- a/prog_test.go +++ b/prog_test.go @@ -662,6 +662,7 @@ func TestProgramSpecCopy(t *testing.T) { "license", 1, binary.LittleEndian, + nil, } qt.Check(t, qt.IsNil((*ProgramSpec)(nil).Copy())) diff --git a/syscalls.go b/syscalls.go index f0f42b77d..edc103f8a 100644 --- a/syscalls.go +++ b/syscalls.go @@ -51,48 +51,63 @@ func sanitizeName(name string, replacement rune) string { }, name) } -func maybeFillObjName(name string) sys.ObjName { - if errors.Is(haveObjName(), ErrNotSupported) { +func maybeFillObjName(name string, tokenFd int32) sys.ObjName { + if errors.Is(haveObjName(internal.WithToken(tokenFd)), ErrNotSupported) { return sys.ObjName{} } name = sanitizeName(name, -1) - if errors.Is(objNameAllowsDot(), ErrNotSupported) { + if errors.Is(objNameAllowsDot(internal.WithToken(tokenFd)), ErrNotSupported) { name = strings.ReplaceAll(name, ".", "") } return sys.NewObjName(name) } -func progLoad(insns asm.Instructions, typ ProgramType, license string) (*sys.FD, error) { +func progLoad(insns asm.Instructions, typ ProgramType, license string, tokenFd int32) (*sys.FD, error) { buf := bytes.NewBuffer(make([]byte, 0, insns.Size())) if err := insns.Marshal(buf, internal.NativeEndian); err != nil { return nil, err } bytecode := buf.Bytes() - return sys.ProgLoad(&sys.ProgLoadAttr{ + attr := &sys.ProgLoadAttr{ ProgType: sys.ProgType(typ), License: sys.NewStringPointer(license), Insns: sys.SlicePointer(bytecode), InsnCnt: uint32(len(bytecode) / asm.InstructionSize), - }) + } + + if tokenFd > 0 { + attr.ProgTokenFd = tokenFd + attr.ProgFlags |= sys.BPF_F_TOKEN_FD + } + + return sys.ProgLoad(attr) } -var haveNestedMaps = internal.NewFeatureTest("nested maps", func() error { +var haveNestedMaps = internal.NewFeatureTest("nested maps", func(opts ...internal.FeatureTestOption) error { if platform.IsWindows { // We only support efW versions which have this feature, no need to probe. return nil } - _, err := sys.MapCreate(&sys.MapCreateAttr{ + attr := &sys.MapCreateAttr{ MapType: sys.MapType(ArrayOfMaps), KeySize: 4, ValueSize: 4, MaxEntries: 1, // Invalid file descriptor. InnerMapFd: ^uint32(0), - }) + } + + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + + _, err := sys.MapCreate(attr) if errors.Is(err, unix.EINVAL) { return internal.ErrNotSupported } @@ -102,16 +117,24 @@ var haveNestedMaps = internal.NewFeatureTest("nested maps", func() error { return err }, "4.12", "windows:0.21.0") -var haveMapMutabilityModifiers = internal.NewFeatureTest("read- and write-only maps", func() error { +var haveMapMutabilityModifiers = internal.NewFeatureTest("read- and write-only maps", func(opts ...internal.FeatureTestOption) error { // This checks BPF_F_RDONLY_PROG and BPF_F_WRONLY_PROG. Since // BPF_MAP_FREEZE appeared in 5.2 as well we don't do a separate check. - m, err := sys.MapCreate(&sys.MapCreateAttr{ + attr := &sys.MapCreateAttr{ MapType: sys.MapType(Array), KeySize: 4, ValueSize: 4, MaxEntries: 1, MapFlags: sys.BPF_F_RDONLY_PROG, - }) + } + + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + + m, err := sys.MapCreate(attr) if err != nil { return internal.ErrNotSupported } @@ -119,15 +142,23 @@ var haveMapMutabilityModifiers = internal.NewFeatureTest("read- and write-only m return nil }, "5.2") -var haveMmapableMaps = internal.NewFeatureTest("mmapable maps", func() error { +var haveMmapableMaps = internal.NewFeatureTest("mmapable maps", func(opts ...internal.FeatureTestOption) error { // This checks BPF_F_MMAPABLE, which appeared in 5.5 for array maps. - m, err := sys.MapCreate(&sys.MapCreateAttr{ + attr := &sys.MapCreateAttr{ MapType: sys.MapType(Array), KeySize: 4, ValueSize: 4, MaxEntries: 1, MapFlags: sys.BPF_F_MMAPABLE, - }) + } + + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + + m, err := sys.MapCreate(attr) if err != nil { return internal.ErrNotSupported } @@ -135,16 +166,23 @@ var haveMmapableMaps = internal.NewFeatureTest("mmapable maps", func() error { return nil }, "5.5") -var haveInnerMaps = internal.NewFeatureTest("inner maps", func() error { +var haveInnerMaps = internal.NewFeatureTest("inner maps", func(opts ...internal.FeatureTestOption) error { // This checks BPF_F_INNER_MAP, which appeared in 5.10. - m, err := sys.MapCreate(&sys.MapCreateAttr{ + attr := &sys.MapCreateAttr{ MapType: sys.MapType(Array), KeySize: 4, ValueSize: 4, MaxEntries: 1, MapFlags: sys.BPF_F_INNER_MAP, - }) + } + + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + m, err := sys.MapCreate(attr) if err != nil { return internal.ErrNotSupported } @@ -152,16 +190,22 @@ var haveInnerMaps = internal.NewFeatureTest("inner maps", func() error { return nil }, "5.10") -var haveNoPreallocMaps = internal.NewFeatureTest("prealloc maps", func() error { +var haveNoPreallocMaps = internal.NewFeatureTest("prealloc maps", func(opts ...internal.FeatureTestOption) error { // This checks BPF_F_NO_PREALLOC, which appeared in 4.6. - m, err := sys.MapCreate(&sys.MapCreateAttr{ + attr := &sys.MapCreateAttr{ MapType: sys.MapType(Hash), KeySize: 4, ValueSize: 4, MaxEntries: 1, MapFlags: sys.BPF_F_NO_PREALLOC, - }) + } + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + m, err := sys.MapCreate(attr) if err != nil { return internal.ErrNotSupported } @@ -193,7 +237,7 @@ func wrapMapError(err error) error { return err } -var haveObjName = internal.NewFeatureTest("object names", func() error { +var haveObjName = internal.NewFeatureTest("object names", func(opts ...internal.FeatureTestOption) error { if platform.IsWindows { // We only support efW versions which have this feature, no need to probe. return nil @@ -207,6 +251,12 @@ var haveObjName = internal.NewFeatureTest("object names", func() error { MapName: sys.NewObjName("feature_test"), } + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + fd, err := sys.MapCreate(&attr) if err != nil { return internal.ErrNotSupported @@ -216,13 +266,13 @@ var haveObjName = internal.NewFeatureTest("object names", func() error { return nil }, "4.15", "windows:0.21.0") -var objNameAllowsDot = internal.NewFeatureTest("dot in object names", func() error { +var objNameAllowsDot = internal.NewFeatureTest("dot in object names", func(opts ...internal.FeatureTestOption) error { if platform.IsWindows { // We only support efW versions which have this feature, no need to probe. return nil } - if err := haveObjName(); err != nil { + if err := haveObjName(opts...); err != nil { return err } @@ -234,6 +284,12 @@ var objNameAllowsDot = internal.NewFeatureTest("dot in object names", func() err MapName: sys.NewObjName(".test"), } + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + fd, err := sys.MapCreate(&attr) if err != nil { return internal.ErrNotSupported @@ -243,7 +299,7 @@ var objNameAllowsDot = internal.NewFeatureTest("dot in object names", func() err return nil }, "5.2", "windows:0.21.0") -var haveBatchAPI = internal.NewFeatureTest("map batch api", func() error { +var haveBatchAPI = internal.NewFeatureTest("map batch api", func(opts ...internal.FeatureTestOption) error { var maxEntries uint32 = 2 attr := sys.MapCreateAttr{ MapType: sys.MapType(Hash), @@ -252,6 +308,12 @@ var haveBatchAPI = internal.NewFeatureTest("map batch api", func() error { MaxEntries: maxEntries, } + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.MapTokenFd = tokenFd + attr.MapFlags |= sys.BPF_F_TOKEN_FD + } + fd, err := sys.MapCreate(&attr) if err != nil { return internal.ErrNotSupported @@ -275,7 +337,7 @@ var haveBatchAPI = internal.NewFeatureTest("map batch api", func() error { return nil }, "5.6") -var haveProbeReadKernel = internal.NewFeatureTest("bpf_probe_read_kernel", func() error { +var haveProbeReadKernel = internal.NewFeatureTest("bpf_probe_read_kernel", func(opts ...internal.FeatureTestOption) error { insns := asm.Instructions{ asm.Mov.Reg(asm.R1, asm.R10), asm.Add.Imm(asm.R1, -8), @@ -285,7 +347,8 @@ var haveProbeReadKernel = internal.NewFeatureTest("bpf_probe_read_kernel", func( asm.Return(), } - fd, err := progLoad(insns, Kprobe, "GPL") + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + fd, err := progLoad(insns, Kprobe, "GPL", tokenFd) if err != nil { return internal.ErrNotSupported } @@ -293,7 +356,7 @@ var haveProbeReadKernel = internal.NewFeatureTest("bpf_probe_read_kernel", func( return nil }, "5.5") -var haveBPFToBPFCalls = internal.NewFeatureTest("bpf2bpf calls", func() error { +var haveBPFToBPFCalls = internal.NewFeatureTest("bpf2bpf calls", func(opts ...internal.FeatureTestOption) error { insns := asm.Instructions{ asm.Call.Label("prog2").WithSymbol("prog1"), asm.Return(), @@ -301,7 +364,8 @@ var haveBPFToBPFCalls = internal.NewFeatureTest("bpf2bpf calls", func() error { asm.Return(), } - fd, err := progLoad(insns, SocketFilter, "MIT") + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + fd, err := progLoad(insns, SocketFilter, "MIT", tokenFd) if err != nil { return internal.ErrNotSupported } @@ -309,7 +373,7 @@ var haveBPFToBPFCalls = internal.NewFeatureTest("bpf2bpf calls", func() error { return nil }, "4.16") -var haveSyscallWrapper = internal.NewFeatureTest("syscall wrapper", func() error { +var haveSyscallWrapper = internal.NewFeatureTest("syscall wrapper", func(...internal.FeatureTestOption) error { prefix := linux.PlatformPrefix() if prefix == "" { return fmt.Errorf("unable to find the platform prefix for (%s)", runtime.GOARCH) @@ -338,7 +402,7 @@ var haveSyscallWrapper = internal.NewFeatureTest("syscall wrapper", func() error return evt.Close() }, "4.17") -var haveProgramExtInfos = internal.NewFeatureTest("program ext_infos", func() error { +var haveProgramExtInfos = internal.NewFeatureTest("program ext_infos", func(opts ...internal.FeatureTestOption) error { insns := asm.Instructions{ asm.Mov.Imm(asm.R0, 0), asm.Return(), @@ -350,15 +414,22 @@ var haveProgramExtInfos = internal.NewFeatureTest("program ext_infos", func() er } bytecode := buf.Bytes() - _, err := sys.ProgLoad(&sys.ProgLoadAttr{ + attr := &sys.ProgLoadAttr{ ProgType: sys.ProgType(SocketFilter), License: sys.NewStringPointer("MIT"), Insns: sys.SlicePointer(bytecode), InsnCnt: uint32(len(bytecode) / asm.InstructionSize), FuncInfoCnt: 1, ProgBtfFd: math.MaxUint32, - }) + } + + tokenFd := internal.BuildOptions(opts...).BpffsTokenFd + if tokenFd > 0 { + attr.ProgTokenFd = tokenFd + attr.ProgFlags |= sys.BPF_F_TOKEN_FD + } + _, err := sys.ProgLoad(attr) if errors.Is(err, unix.EBADF) { return nil } diff --git a/types.go b/types.go index 52ff75b5c..367df7036 100644 --- a/types.go +++ b/types.go @@ -364,6 +364,8 @@ type LoadPinOptions struct { // Raw flags for the syscall. Other fields of this struct take precedence. Flags uint32 + + BpffsTokenFd *sys.FD } // Marshal returns a value suitable for BPF_OBJ_GET syscall file_flags parameter.