Merge pull request #201562 from jdbaldry/add/osquery

This commit is contained in:
Ryan Lahfa 2023-07-20 11:06:46 +02:00 committed by GitHub
commit 2d0a128ac9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 498 additions and 1 deletions

View file

@ -18466,6 +18466,12 @@
github = "zmitchell";
githubId = 10246891;
};
znewman01 = {
email = "znewman01@gmail.com";
github = "znewman01";
githubId = 873857;
name = "Zack Newman";
};
zoedsoupe = {
github = "zoedsoupe";
githubId = 44469426;

View file

@ -26,6 +26,8 @@
- [trust-dns](https://trust-dns.org/), a Rust based DNS server built to be safe and secure from the ground up. Available as [services.trust-dns](#opt-services.trust-dns.enable).
- [osquery](https://www.osquery.io/), a SQL powered operating system instrumentation, monitoring, and analytics.
## Backward Incompatibilities {#sec-release-23.11-incompatibilities}
- The `boot.loader.raspberryPi` options have been marked deprecated, with intent for removal for NixOS 24.11. They had a limited use-case, and do not work like people expect. They required either very old installs ([before mid-2019](https://github.com/NixOS/nixpkgs/pull/62462)) or customized builds out of scope of the standard and generic AArch64 support. That option set never supported the Raspberry Pi 4 family of devices.

View file

@ -764,6 +764,7 @@
./services/monitoring/nagios.nix
./services/monitoring/netdata.nix
./services/monitoring/opentelemetry-collector.nix
./services/monitoring/osquery.nix
./services/monitoring/parsedmarc.nix
./services/monitoring/prometheus/alertmanager-irc-relay.nix
./services/monitoring/prometheus/alertmanager.nix

View file

@ -72,7 +72,6 @@ in
(mkRemovedOptionModule [ "services" "mesos" ] "The corresponding package was removed from nixpkgs.")
(mkRemovedOptionModule [ "services" "moinmoin" ] "The corresponding package was removed from nixpkgs.")
(mkRemovedOptionModule [ "services" "mwlib" ] "The corresponding package was removed from nixpkgs.")
(mkRemovedOptionModule [ "services" "osquery" ] "The osquery module has been removed")
(mkRemovedOptionModule [ "services" "pantheon" "files" ] ''
This module was removed, please add pkgs.pantheon.elementary-files to environment.systemPackages directly.
'')

View file

@ -0,0 +1,97 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.osquery;
dirname = path: with lib.strings; with lib.lists; concatStringsSep "/"
(init (splitString "/" (normalizePath path)));
# conf is the osquery configuration file used when the --config_plugin=filesystem.
# filesystem is the osquery default value for the config_plugin flag.
conf = pkgs.writeText "osquery.conf" (builtins.toJSON cfg.settings);
# flagfile is the file containing osquery command line flags to be
# provided to the application using the special --flagfile option.
flagfile = pkgs.writeText "osquery.flags"
(concatStringsSep "\n"
(mapAttrsToList (name: value: "--${name}=${value}")
# Use the conf derivation if not otherwise specified.
({ config_path = conf; } // cfg.flags)));
osqueryi = pkgs.runCommand "osqueryi" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
mkdir -p $out/bin
makeWrapper ${pkgs.osquery}/bin/osqueryi $out/bin/osqueryi \
--add-flags "--flagfile ${flagfile}"
'';
in
{
options.services.osquery = {
enable = mkEnableOption (mdDoc "osqueryd daemon");
settings = mkOption {
default = { };
description = mdDoc ''
Configuration to be written to the osqueryd JSON configuration file.
To understand the configuration format, refer to https://osquery.readthedocs.io/en/stable/deployment/configuration/#configuration-components.
'';
example = {
options.utc = false;
};
type = types.attrs;
};
flags = mkOption {
default = { };
description = mdDoc ''
Attribute set of flag names and values to be written to the osqueryd flagfile.
For more information, refer to https://osquery.readthedocs.io/en/stable/installation/cli-flags.
'';
example = {
config_refresh = "10";
};
type = with types;
submodule {
freeformType = attrsOf str;
options = {
database_path = mkOption {
default = "/var/lib/osquery/osquery.db";
readOnly = true;
description = mdDoc "Path used for the database file.";
type = path;
};
logger_path = mkOption {
default = "/var/log/osquery";
readOnly = true;
description = mdDoc "Base directory used for logging.";
type = path;
};
pidfile = mkOption {
default = "/run/osquery/osqueryd.pid";
readOnly = true;
description = mdDoc "Path used for pid file.";
type = path;
};
};
};
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ osqueryi ];
systemd.services.osqueryd = {
after = [ "network.target" "syslog.service" ];
description = "The osquery daemon";
serviceConfig = {
ExecStart = "${pkgs.osquery}/bin/osqueryd --flagfile ${flagfile}";
PIDFile = cfg.flags.pidfile;
LogsDirectory = cfg.flags.logger_path;
StateDirectory = dirname cfg.flags.database_path;
Restart = "always";
};
wantedBy = [ "multi-user.target" ];
};
systemd.tmpfiles.rules = [
"d ${dirname (cfg.flags.pidfile)} 0755 root root -"
];
};
}

View file

@ -577,6 +577,7 @@ in {
openvscode-server = handleTest ./openvscode-server.nix {};
orangefs = handleTest ./orangefs.nix {};
os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {};
osquery = handleTestOn ["x86_64-linux"] ./osquery.nix {};
osrm-backend = handleTest ./osrm-backend.nix {};
overlayfs = handleTest ./overlayfs.nix {};
pacemaker = handleTest ./pacemaker.nix {};

56
nixos/tests/osquery.nix Normal file
View file

@ -0,0 +1,56 @@
import ./make-test-python.nix ({ lib, pkgs, ... }:
with lib;
let
config_refresh = "10";
nullvalue = "NULL";
utc = false;
in
{
name = "osquery";
meta = with maintainers; {
maintainers = [ znewman01 lewo ];
};
nodes.machine = { config, pkgs, ... }: {
services.osquery = {
enable = true;
settings.options = { inherit nullvalue utc; };
flags = {
inherit config_refresh;
nullvalue = "IGNORED";
};
};
};
testScript = { nodes, ... }:
let
cfg = nodes.machine.services.osquery;
in
''
machine.start()
machine.wait_for_unit("osqueryd.service")
# Stop the osqueryd service so that we can use osqueryi to check information stored in the database.
machine.wait_until_succeeds("systemctl stop osqueryd.service")
# osqueryd was able to query information about the host.
machine.succeed("echo 'SELECT address FROM etc_hosts LIMIT 1;' | osqueryi | tee /dev/console | grep -q '127.0.0.1'")
# osquery binaries respect configuration from the Nix config option.
machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"utc\";' | osqueryi | tee /dev/console | grep -q ${boolToString utc}")
# osquery binaries respect configuration from the Nix flags option.
machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"config_refresh\";' | osqueryi | tee /dev/console | grep -q ${config_refresh}")
# Demonstrate that osquery binaries prefer configuration plugin options over CLI flags.
# https://osquery.readthedocs.io/en/latest/deployment/configuration/#options.
machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"nullvalue\";' | osqueryi | tee /dev/console | grep -q ${nullvalue}")
# Module creates directories for default database_path and pidfile flag values.
machine.succeed("test -d $(dirname ${cfg.flags.database_path})")
machine.succeed("test -d $(dirname ${cfg.flags.pidfile})")
'';
})

View file

@ -0,0 +1,25 @@
From: Jack Baldry <jack.baldry@grafana.com>
Date: Tue, 15 Nov 2022 15:40:31 -0400
Subject: [PATCH] Remove circular definition of AUDIT_FILTER_EXCLUDE
https://github.com/osquery/osquery/issues/6551
Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
---
libraries/cmake/source/libaudit/src/lib/libaudit.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/libraries/cmake/source/libaudit/src/lib/libaudit.h b/libraries/cmake/source/libaudit/src/libaudit.h
--- a/libraries/cmake/source/libaudit/src/lib/libaudit.h
+++ b/libraries/cmake/source/libaudit/src/lib/libaudit.h
@@ -260,7 +260,6 @@ extern "C" {
#define AUDIT_KEY_SEPARATOR 0x01
/* These are used in filter control */
-#define AUDIT_FILTER_EXCLUDE AUDIT_FILTER_TYPE
#define AUDIT_FILTER_MASK 0x07 /* Mask to get actual filter */
#define AUDIT_FILTER_UNSET 0x80 /* This value means filter is unset */
--
2.38.1

View file

@ -0,0 +1,37 @@
From: Jack Baldry <jack.baldry@grafana.com>
Date: Tue, 15 Nov 2022 13:48:07 -0400
Subject: [PATCH] Remove git reset
This is not required for nixpkgs builds because we are not working in
the source repository and therefore do not need to be careful about
updating submodule content.
Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
---
libraries/cmake/source/modules/utils.cmake | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/libraries/cmake/source/modules/utils.cmake b/libraries/cmake/source/modules/utils.cmake
--- a/libraries/cmake/source/modules/utils.cmake
+++ b/libraries/cmake/source/modules/utils.cmake
@@ -102,17 +102,6 @@ function(patchSubmoduleSourceCode library_name patches_dir source_dir apply_to_d
file(COPY "${source_dir}" DESTINATION "${parent_dir}")
endif()
- # We need to restore the source code to its original state, pre patch
- execute_process(
- COMMAND "${GIT_EXECUTABLE}" reset --hard HEAD
- RESULT_VARIABLE process_exit_code
- WORKING_DIRECTORY "${source_dir}"
- )
-
- if(NOT ${process_exit_code} EQUAL 0)
- message(FATAL_ERROR "Failed to git reset the following submodule: \"${source_dir}\"")
- endif()
-
set(patchSubmoduleSourceCode_Patched TRUE PARENT_SCOPE)
endfunction()
--
2.38.1

View file

@ -0,0 +1,157 @@
From: Jack Baldry <jack.baldry@grafana.com>
Date: Wed, 16 Nov 2022 22:00:06 -0400
Subject: [PATCH] Remove system controls table
Relies on <sys/sysctl.h> which is not present in glibc since 2.32.
Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
---
osquery/tables/system/CMakeLists.txt | 4 --
specs/CMakeLists.txt | 1 -
specs/posix/system_controls.table | 21 -------
tests/integration/tables/system_controls.cpp | 61 --------------------
4 files changed, 87 deletions(-)
delete mode 100644 specs/posix/system_controls.table
delete mode 100644 tests/integration/tables/system_controls.cpp
diff --git a/osquery/tables/system/CMakeLists.txt b/osquery/tables/system/CMakeLists.txt
--- a/osquery/tables/system/CMakeLists.txt
+++ b/osquery/tables/system/CMakeLists.txt
@@ -43,7 +43,6 @@ function(generateOsqueryTablesSystemSystemtable)
posix/smbios_utils.cpp
posix/sudoers.cpp
posix/suid_bin.cpp
- posix/system_controls.cpp
posix/ulimit_info.cpp
)
endif()
@@ -82,7 +81,6 @@ function(generateOsqueryTablesSystemSystemtable)
linux/shared_memory.cpp
linux/smbios_tables.cpp
linux/startup_items.cpp
- linux/sysctl_utils.cpp
linux/system_info.cpp
linux/usb_devices.cpp
linux/user_groups.cpp
@@ -156,7 +154,6 @@ function(generateOsqueryTablesSystemSystemtable)
darwin/smbios_tables.cpp
darwin/smc_keys.cpp
darwin/startup_items.cpp
- darwin/sysctl_utils.cpp
darwin/system_extensions.mm
darwin/system_info.cpp
darwin/time_machine.cpp
@@ -326,7 +323,6 @@ function(generateOsqueryTablesSystemSystemtable)
posix/shell_history.h
posix/ssh_keys.h
posix/sudoers.h
- posix/sysctl_utils.h
posix/last.h
posix/openssl_utils.h
posix/authorized_keys.h
diff --git a/specs/CMakeLists.txt b/specs/CMakeLists.txt
--- a/specs/CMakeLists.txt
+++ b/specs/CMakeLists.txt
@@ -246,7 +246,6 @@ function(generateNativeTables)
"posix/socket_events.table:linux,macos"
"posix/sudoers.table:linux,macos,freebsd"
"posix/suid_bin.table:linux,macos,freebsd"
- "posix/system_controls.table:linux,macos,freebsd"
"posix/ulimit_info.table:linux,macos,freebsd"
"posix/usb_devices.table:linux,macos"
"posix/user_events.table:linux,macos,freebsd"
diff --git a/specs/posix/system_controls.table b/specs/posix/system_controls.table
deleted file mode 100644
--- a/specs/posix/system_controls.table
+++ /dev/null
@@ -1,21 +0,0 @@
-table_name("system_controls")
-description("sysctl names, values, and settings information.")
-schema([
- Column("name", TEXT, "Full sysctl MIB name", index=True),
- Column("oid", TEXT, "Control MIB", additional=True),
- Column("subsystem", TEXT, "Subsystem ID, control type", additional=True),
- Column("current_value", TEXT, "Value of setting"),
- Column("config_value", TEXT, "The MIB value set in /etc/sysctl.conf"),
- Column("type", TEXT, "Data type"),
-])
-extended_schema(DARWIN, [
- Column("field_name", TEXT, "Specific attribute of opaque type"),
-])
-
-implementation("system_controls@genSystemControls")
-fuzz_paths([
- "/run/sysctl.d/",
- "/usr/lib/sysctl.d/",
- "/lib/sysctl.d/",
- "/sys"
-])
diff --git a/tests/integration/tables/system_controls.cpp b/tests/integration/tables/system_controls.cpp
deleted file mode 100644
--- a/tests/integration/tables/system_controls.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Copyright (c) 2014-present, The osquery authors
- *
- * This source code is licensed as defined by the LICENSE file found in the
- * root directory of this source tree.
- *
- * SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
- */
-
-// Sanity check integration test for system_controls
-// Spec file: specs/posix/system_controls.table
-
-#include <osquery/tests/integration/tables/helper.h>
-
-namespace osquery {
-namespace table_tests {
-namespace {
-
-class SystemControlsTest : public testing::Test {
- protected:
- void SetUp() override {
- setUpEnvironment();
- }
-};
-
-TEST_F(SystemControlsTest, test_sanity) {
- auto const rows = execute_query("select * from system_controls");
- auto const row_map = ValidationMap{
- {"name", NonEmptyString},
- {"oid", NormalType},
- {"subsystem",
- SpecificValuesCheck{"",
- "abi",
- "debug",
- "dev",
- "fs",
- "fscache",
- "hw",
- "kern",
- "kernel",
- "machdep",
- "net",
- "sunrpc",
- "user",
- "vfs",
- "vm"}},
- {"current_value", NormalType},
- {"config_value", NormalType},
- {"type",
- SpecificValuesCheck{
- "", "node", "int", "string", "quad", "opaque", "struct"}},
-#ifdef __APPLE__
- {"field_name", NormalType},
-#endif
- };
- validate_rows(rows, row_map);
-}
-
-} // namespace
-} // namespace table_tests
-} // namespace osquery
--
2.38.1

View file

@ -0,0 +1,29 @@
From: Jack Baldry <jack.baldry@grafana.com>
Date: Tue, 15 Nov 2022 14:34:33 -0400
Subject: [PATCH] Use locale.h instead of removed xlocale.h header
https://sourceware.org/glibc/wiki/Release/2.26#Removal_of_.27xlocale.h.27
Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
---
libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
--- a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
+++ b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
@@ -48,9 +48,9 @@
/* NetBSD 5.0 mis-defines NULL. */
#include <stddef.h>
-/* Mac OS X 10.5 defines the locale_t type in <xlocale.h>. */
+/* Mac OS X 10.5 defines the locale_t type in <locale.h>. */
#if 1
-# include <xlocale.h>
+# include <locale.h>
#endif
/* The definitions of _GL_FUNCDECL_RPL etc. are copied here. */
--
2.38.1

View file

@ -0,0 +1,85 @@
{ lib
, cmake
, fetchFromGitHub
, git
, llvmPackages
, nixosTests
, overrideCC
, perl
, python3
, stdenv
, openssl_1_1
}:
let
buildStdenv = overrideCC stdenv llvmPackages.clangUseLLVM;
in
buildStdenv.mkDerivation rec {
pname = "osquery";
version = "5.5.1";
src = fetchFromGitHub {
owner = "osquery";
repo = "osquery";
rev = version;
fetchSubmodules = true;
sha256 = "sha256-Q6PQVnBjAjAlR725fyny+RhQFUNwxWGjLDuS5p9JKlU=";
};
patches = [
./Remove-git-reset.patch
./Use-locale.h-instead-of-removed-xlocale.h-header.patch
./Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch
# For current state of compilation against glibc in the clangWithLLVM toolchain, refer to the upstream issue in https://github.com/osquery/osquery/issues/7823.
./Remove-system-controls-table.patch
];
buildInputs = [
llvmPackages.libunwind
];
nativeBuildInputs = [
cmake
git
perl
python3
];
postPatch = ''
substituteInPlace cmake/install_directives.cmake --replace "/control" "control"
# This is required to build libarchive with our glibc version
# which provides the ARC4RANDOM_BUF function
substituteInPlace libraries/cmake/source/libarchive/CMakeLists.txt --replace " target_compile_definitions(thirdparty_libarchive PRIVATE" " target_compile_definitions(thirdparty_libarchive PRIVATE HAVE_ARC4RANDOM_BUF"
# We need to override this hash because we use our own openssl 1.1 version
substituteInPlace libraries/cmake/formula/openssl/CMakeLists.txt --replace "d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca" "e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6"
cat libraries/cmake/formula/openssl/CMakeLists.txt
'';
# For explanation of these deletions, refer to the ./Use-locale.h-instead-of-removed-xlocale.h-header.patch file.
preConfigure = ''
find libraries/cmake/source -name 'config.h' -exec sed -i '/#define HAVE_XLOCALE_H 1/d' {} \;
'';
cmakeFlags = [
"-DOSQUERY_VERSION=${version}"
"-DOSQUERY_OPENSSL_ARCHIVE_PATH=${openssl_1_1.src}"
];
postFixup = ''
patchelf --set-rpath "${llvmPackages.libunwind}/lib:$(patchelf --print-rpath $out/bin/osqueryd)" "$out/bin/osqueryd"
'';
passthru.tests.osquery = nixosTests.osquery;
meta = with lib; {
description = "SQL powered operating system instrumentation, monitoring, and analytics.";
longDescription = ''
The system controls table is not included as it does not presently compile with glibc >= 2.32.
For more information, refer to https://github.com/osquery/osquery/issues/7823
'';
homepage = "https://osquery.io";
license = licenses.bsd3;
platforms = platforms.linux;
maintainers = with maintainers; [ znewman01 lewo ];
};
}

View file

@ -1789,6 +1789,8 @@ with pkgs;
openbugs = pkgsi686Linux.callPackage ../applications/science/machine-learning/openbugs { };
osquery = callPackage ../tools/system/osquery { };
paperview = callPackage ../tools/X11/paperview { };
pferd = callPackage ../tools/misc/pferd { };