Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

introduce file system watching features to the zig build system #20580

Merged
merged 38 commits into from
Jul 12, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift click to select a range
908c2c9
std.Build.Cache.Path: add eql method
andrewrk Jul 5, 2024
d2bec8f
delete dead CLI usage code from main.zig
andrewrk Jul 5, 2024
6e025fc
build system: add --watch flag and report source file in InstallFile
andrewrk Jul 5, 2024
26d506c
std.Build: remove the "push installed file" mechanism
andrewrk Jul 5, 2024
6c64090
std.os.linux: fanotify_init, fanotify_mark, name_to_handle_at
andrewrk Jul 9, 2024
deea362
std.Build.Cache.Path: add `subPathOpt` and `TableAdapter`
andrewrk Jul 9, 2024
bbd90a5
build runner: implement --watch (work-in-progress)
andrewrk Jul 9, 2024
c5a4177
std.os.linux: add AT.HANDLE_FID
andrewrk Jul 10, 2024
5ee3971
proof-of-concept --watch implementation based on fanotify
andrewrk Jul 10, 2024
6f89824
build system: make debounce interval CLI-configurable
andrewrk Jul 10, 2024
001ff7b
std.Build.Watch: make dirty steps invalidate each other
andrewrk Jul 10, 2024
e6b6a72
build runner: fix build summary painting over CLI progress
andrewrk Jul 10, 2024
e712ca5
std.os.linux: type safety for fanotify metadata event mask field
andrewrk Jul 10, 2024
956f1eb
std.Build.Watch: gracefully handle fanotify queue overflow
andrewrk Jul 10, 2024
0cc492a
make more build steps integrate with the watch system
andrewrk Jul 10, 2024
d1c14f2
std.Build.Step.WriteFile: extract UpdateSourceFiles
andrewrk Jul 10, 2024
dcbb3aa
std.Build.Cache.Path: fix format function for absolute paths
andrewrk Jul 10, 2024
0994e22
build runner: more useful failure handling for fanotify_mark
andrewrk Jul 10, 2024
26bdc83
std.Build.LazyPath: add getPath3; deprecate getPath2 and getPath
andrewrk Jul 10, 2024
7bccef3
std.Build.Watch: introduce special file "." to watch entire dir
andrewrk Jul 10, 2024
2ebf021
build runner: don't pass a dirfd null to fanotify_mark
andrewrk Jul 10, 2024
b6ed833
build runner: ignore ENOENT of fanotify_mark REMOVE
andrewrk Jul 10, 2024
61d2234
std.Build.Watch: add ONDIR to fanotify event mask
andrewrk Jul 10, 2024
5c3fae3
td.Build.Step.InstallDir: leave hint for wrong cached status
andrewrk Jul 10, 2024
f285640
introduce std.Build.Cache.Manifest.addFilePath
andrewrk Jul 10, 2024
a966eee
std.Build.Step.WriteFile: fix handling of directories
andrewrk Jul 10, 2024
6fcb189
std.Build.Step.WriteFile: remove random bytes from cache hash
andrewrk Jul 11, 2024
2e42969
std.Build.Step.Run: integrate with --watch
andrewrk Jul 11, 2024
dad07fb
std.Build.Cache.Path: fix hash impl on windows
andrewrk Jul 11, 2024
768cb7e
objcopy: use the fatal helper method
andrewrk Jul 11, 2024
818f9cb
std.Build.Step.ObjCopy: remove random bytes from cache hash
andrewrk Jul 11, 2024
fd4d366
std.Build.Cache.Path: fix the format method
andrewrk Jul 11, 2024
a3c20df
integrate Compile steps with file watching
andrewrk Jul 11, 2024
5a34e6c
frontend: add file system inputs for incremental cache mode
andrewrk Jul 12, 2024
67e3e49
Compilation: fix rebase conflict
andrewrk Jul 12, 2024
4f9a8b6
update build system unit test
andrewrk Jul 12, 2024
f77b43d
zig build: add a --debug-target CLI flag
andrewrk Jul 12, 2024
5efcc2e
build runner: refactor fs watch logic for OS abstraction
andrewrk Jul 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
std.os.linux: fanotify_init, fanotify_mark, name_to_handle_at
* Delete existing `FAN` struct in favor of a `fanotify` struct which has
  type-safe bindings (breaking).
* Add name_to_handle_at syscall wrapper.
* Add file_handle
* Add kernel_fsid_t
* Add fsid_t
* Add and update std.posix wrappers.
  • Loading branch information
andrewrk committed Jul 12, 2024
commit 6c64090e7af56ccbf374e1a927fe232b340ed681
232 changes: 180 additions & 52 deletions lib/std/os/linux.zig
Original file line number Diff line number Diff line change
Expand Up @@ -698,12 698,42 @@ pub fn inotify_rm_watch(fd: i32, wd: i32) usize {
return syscall2(.inotify_rm_watch, @as(usize, @bitCast(@as(isize, fd))), @as(usize, @bitCast(@as(isize, wd))));
}

pub fn fanotify_init(flags: u32, event_f_flags: u32) usize {
return syscall2(.fanotify_init, flags, event_f_flags);
pub fn fanotify_init(flags: fanotify.InitFlags, event_f_flags: u32) usize {
return syscall2(.fanotify_init, @as(u32, @bitCast(flags)), event_f_flags);
}

pub fn fanotify_mark(fd: i32, flags: u32, mask: u64, dirfd: i32, pathname: ?[*:0]const u8) usize {
return syscall5(.fanotify_mark, @as(usize, @bitCast(@as(isize, fd))), flags, mask, @as(usize, @bitCast(@as(isize, dirfd))), @intFromPtr(pathname));
pub fn fanotify_mark(
fd: fd_t,
flags: fanotify.MarkFlags,
mask: fanotify.MarkMask,
dirfd: fd_t,
pathname: ?[*:0]const u8,
) usize {
return syscall5(
.fanotify_mark,
@bitCast(@as(isize, fd)),
@as(u32, @bitCast(flags)),
@bitCast(mask),
@bitCast(@as(isize, dirfd)),
@intFromPtr(pathname),
);
}

pub fn name_to_handle_at(
dirfd: fd_t,
pathname: [*:0]const u8,
handle: *std.os.linux.file_handle,
mount_id: *i32,
flags: u32,
) usize {
return syscall5(
.name_to_handle_at,
@as(u32, @bitCast(dirfd)),
@intFromPtr(pathname),
@intFromPtr(handle),
@intFromPtr(mount_id),
flags,
);
}

pub fn readlink(noalias path: [*:0]const u8, noalias buf_ptr: [*]u8, buf_len: usize) usize {
Expand Down Expand Up @@ -4135,58 4165,156 @@ pub const IN = struct {
pub const ONESHOT = 0x80000000;
};

pub const FAN = struct {
pub const ACCESS = 0x00000001;
pub const MODIFY = 0x00000002;
pub const CLOSE_WRITE = 0x00000008;
pub const CLOSE_NOWRITE = 0x00000010;
pub const OPEN = 0x00000020;
pub const Q_OVERFLOW = 0x00004000;
pub const OPEN_PERM = 0x00010000;
pub const ACCESS_PERM = 0x00020000;
pub const ONDIR = 0x40000000;
pub const EVENT_ON_CHILD = 0x08000000;
pub const CLOSE = CLOSE_WRITE | CLOSE_NOWRITE;
pub const CLOEXEC = 0x00000001;
pub const NONBLOCK = 0x00000002;
pub const CLASS_NOTIF = 0x00000000;
pub const CLASS_CONTENT = 0x00000004;
pub const CLASS_PRE_CONTENT = 0x00000008;
pub const ALL_CLASS_BITS = CLASS_NOTIF | CLASS_CONTENT | CLASS_PRE_CONTENT;
pub const UNLIMITED_QUEUE = 0x00000010;
pub const UNLIMITED_MARKS = 0x00000020;
pub const ALL_INIT_FLAGS = CLOEXEC | NONBLOCK | ALL_CLASS_BITS | UNLIMITED_QUEUE | UNLIMITED_MARKS;
pub const MARK_ADD = 0x00000001;
pub const MARK_REMOVE = 0x00000002;
pub const MARK_DONT_FOLLOW = 0x00000004;
pub const MARK_ONLYDIR = 0x00000008;
pub const MARK_MOUNT = 0x00000010;
pub const MARK_IGNORED_MASK = 0x00000020;
pub const MARK_IGNORED_SURV_MODIFY = 0x00000040;
pub const MARK_FLUSH = 0x00000080;
pub const ALL_MARK_FLAGS = MARK_ADD | MARK_REMOVE | MARK_DONT_FOLLOW | MARK_ONLYDIR | MARK_MOUNT | MARK_IGNORED_MASK | MARK_IGNORED_SURV_MODIFY | MARK_FLUSH;
pub const ALL_EVENTS = ACCESS | MODIFY | CLOSE | OPEN;
pub const ALL_PERM_EVENTS = OPEN_PERM | ACCESS_PERM;
pub const ALL_OUTGOING_EVENTS = ALL_EVENTS | ALL_PERM_EVENTS | Q_OVERFLOW;
pub const ALLOW = 0x01;
pub const DENY = 0x02;
};

pub const fanotify_event_metadata = extern struct {
event_len: u32,
vers: u8,
reserved: u8,
metadata_len: u16,
mask: u64 align(8),
fd: i32,
pid: i32,
pub const fanotify = struct {
pub const InitFlags = packed struct(u32) {
CLOEXEC: bool = false,
NONBLOCK: bool = false,
CLASS: enum(u2) {
NOTIF = 0,
CONTENT = 1,
PRE_CONTENT = 2,
} = .NOTIF,
UNLIMITED_QUEUE: bool = false,
UNLIMITED_MARKS: bool = false,
ENABLE_AUDIT: bool = false,
REPORT_PIDFD: bool = false,
REPORT_TID: bool = false,
REPORT_FID: bool = false,
REPORT_DIR_FID: bool = false,
REPORT_NAME: bool = false,
REPORT_TARGET_FID: bool = false,
_: u19 = 0,
};

pub const MarkFlags = packed struct(u32) {
ADD: bool = false,
REMOVE: bool = false,
DONT_FOLLOW: bool = false,
ONLYDIR: bool = false,
MOUNT: bool = false,
/// Mutually exclusive with `IGNORE`
IGNORED_MASK: bool = false,
IGNORED_SURV_MODIFY: bool = false,
FLUSH: bool = false,
FILESYSTEM: bool = false,
EVICTABLE: bool = false,
/// Mutually exclusive with `IGNORED_MASK`
IGNORE: bool = false,
_: u21 = 0,
};

pub const MarkMask = packed struct(u64) {
/// File was accessed
ACCESS: bool = false,
/// File was modified
MODIFY: bool = false,
/// Metadata changed
ATTRIB: bool = false,
/// Writtable file closed
CLOSE_WRITE: bool = false,
/// Unwrittable file closed
CLOSE_NOWRITE: bool = false,
/// File was opened
OPEN: bool = false,
/// File was moved from X
MOVED_FROM: bool = false,
/// File was moved to Y
MOVED_TO: bool = false,

/// Subfile was created
CREATE: bool = false,
/// Subfile was deleted
DELETE: bool = false,
/// Self was deleted
DELETE_SELF: bool = false,
/// Self was moved
MOVE_SELF: bool = false,
/// File was opened for exec
OPEN_EXEC: bool = false,
reserved13: u1 = 0,
/// Event queued overflowed
Q_OVERFLOW: bool = false,
/// Filesystem error
FS_ERROR: bool = false,

/// File open in perm check
OPEN_PERM: bool = false,
/// File accessed in perm check
ACCESS_PERM: bool = false,
/// File open/exec in perm check
OPEN_EXEC_PERM: bool = false,
reserved19: u8 = 0,
/// Interested in child events
EVENT_ON_CHILD: bool = false,
/// File was renamed
RENAME: bool = false,
reserved30: u1 = 0,
/// Event occurred against dir
ONDIR: bool = false,
reserved31: u33 = 0,
};

pub const event_metadata = extern struct {
event_len: u32,
vers: u8,
reserved: u8,
metadata_len: u16,
mask: u64 align(8),
fd: i32,
pid: i32,

pub const VERSION = 3;
};

pub const response = extern struct {
fd: i32,
response: u32,
};

/// Unique file identifier info record.
///
/// This structure is used for records of types `EVENT_INFO_TYPE.FID`.
/// `EVENT_INFO_TYPE.DFID` and `EVENT_INFO_TYPE.DFID_NAME`.
///
/// For `EVENT_INFO_TYPE.DFID_NAME` there is additionally a null terminated
/// name immediately after the file handle.
pub const event_info_fid = extern struct {
hdr: event_info_header,
fsid: kernel_fsid_t,
/// Following is an opaque struct file_handle that can be passed as
/// an argument to open_by_handle_at(2).
handle: [0]u8,
};

/// Variable length info record following event metadata.
pub const event_info_header = extern struct {
info_type: EVENT_INFO_TYPE,
pad: u8,
len: u16,
};

pub const EVENT_INFO_TYPE = enum(u8) {
FID = 1,
DFID_NAME = 2,
DFID = 3,
PIDFD = 4,
ERROR = 5,
OLD_DFID_NAME = 10,
OLD_DFID = 11,
NEW_DFID_NAME = 12,
NEW_DFID = 13,
};
};

pub const fanotify_response = extern struct {
fd: i32,
response: u32,
pub const file_handle = extern struct {
handle_bytes: u32,
handle_type: i32,
f_handle: [0]u8,
};

pub const kernel_fsid_t = fsid_t;
pub const fsid_t = [2]i32;

pub const S = struct {
pub const IFMT = 0o170000;

Expand Down
60 changes: 55 additions & 5 deletions lib/std/posix.zig
Original file line number Diff line number Diff line change
Expand Up @@ -4501,7 4501,7 @@ pub const FanotifyInitError = error{
PermissionDenied,
} || UnexpectedError;

pub fn fanotify_init(flags: u32, event_f_flags: u32) FanotifyInitError!i32 {
pub fn fanotify_init(flags: std.os.linux.fanotify.InitFlags, event_f_flags: u32) FanotifyInitError!i32 {
const rc = system.fanotify_init(flags, event_f_flags);
switch (errno(rc)) {
.SUCCESS => return @intCast(rc),
Expand Down Expand Up @@ -4530,16 4530,28 @@ pub const FanotifyMarkError = error{
NameTooLong,
} || UnexpectedError;

pub fn fanotify_mark(fanotify_fd: i32, flags: u32, mask: u64, dirfd: i32, pathname: ?[]const u8) FanotifyMarkError!void {
pub fn fanotify_mark(
fanotify_fd: fd_t,
flags: std.os.linux.fanotify.MarkFlags,
mask: std.os.linux.fanotify.MarkMask,
dirfd: fd_t,
pathname: ?[]const u8,
) FanotifyMarkError!void {
if (pathname) |path| {
const path_c = try toPosixPath(path);
return fanotify_markZ(fanotify_fd, flags, mask, dirfd, &path_c);
} else {
return fanotify_markZ(fanotify_fd, flags, mask, dirfd, null);
}

return fanotify_markZ(fanotify_fd, flags, mask, dirfd, null);
}

pub fn fanotify_markZ(fanotify_fd: i32, flags: u32, mask: u64, dirfd: i32, pathname: ?[*:0]const u8) FanotifyMarkError!void {
pub fn fanotify_markZ(
fanotify_fd: fd_t,
flags: std.os.linux.fanotify.MarkFlags,
mask: std.os.linux.fanotify.MarkMask,
dirfd: fd_t,
pathname: ?[*:0]const u8,
) FanotifyMarkError!void {
const rc = system.fanotify_mark(fanotify_fd, flags, mask, dirfd, pathname);
switch (errno(rc)) {
.SUCCESS => return,
Expand Down Expand Up @@ -7274,6 7286,44 @@ pub fn ptrace(request: u32, pid: pid_t, addr: usize, signal: usize) PtraceError!
};
}

pub const NameToFileHandleAtError = error{
FileNotFound,
NotDir,
OperationNotSupported,
NameTooLong,
Unexpected,
};

pub fn name_to_handle_at(
dirfd: fd_t,
pathname: []const u8,
handle: *std.os.linux.file_handle,
mount_id: *i32,
flags: u32,
) NameToFileHandleAtError!void {
const pathname_c = try toPosixPath(pathname);
return name_to_handle_atZ(dirfd, &pathname_c, handle, mount_id, flags);
}

pub fn name_to_handle_atZ(
dirfd: fd_t,
pathname_z: [*:0]const u8,
handle: *std.os.linux.file_handle,
mount_id: *i32,
flags: u32,
) NameToFileHandleAtError!void {
switch (errno(system.name_to_handle_at(dirfd, pathname_z, handle, mount_id, flags))) {
.SUCCESS => {},
.FAULT => unreachable, // pathname, mount_id, or handle outside accessible address space
.INVAL => unreachable, // bad flags, or handle_bytes too big
.NOENT => return error.FileNotFound,
.NOTDIR => return error.NotDir,
.OPNOTSUPP => return error.OperationNotSupported,
.OVERFLOW => return error.NameTooLong,
else => |err| return unexpectedErrno(err),
}
}

pub const IoCtl_SIOCGIFINDEX_Error = error{
FileSystem,
InterfaceNotFound,
Expand Down