feat: re-think how I collate the data

Vibe-coding being stupid strikes again.

plots/straight-data.gnuplot

@@ -22,7 +22,7 @@ set multiplot
 set yrange [-0.1:2 < * < 10] extend
 set xrange [-1000:30000] extend
 unset ytics
-plot "data/1066528_acceleration.txt" using 1:(dmag($2,$3,$4)) w l t "Motion" lw 1 lc "gray"
+plot "data/motion/1066528_acceleration.txt" using 1:(dmag($2,$3,$4)) w l t "Motion" lw 1 lc "gray"
 
 unset title
 unset xtics
@@ -30,8 +30,8 @@ unset border
 
 set yrange [-2:8]
 set ytics ("Wake" 0, "N1" 1, "N2" 2, "N3" 3, "REM" 5)
-plot "data/1066528_labeled_sleep.txt" using 1:2 w l t "Sleep" lw 3 lc "blue"
+plot "data/labels/1066528_labeled_sleep.txt" using 1:2 w l t "Sleep" lw 3 lc "blue"
 
 set y2range [0:200]
 set datafile separator ","
-plot "data/1066528_heartrate.txt" using 1:2 w l axes x1y2 t "HR" lc "red"
+plot "data/heart_rate/1066528_heartrate.txt" using 1:2 w l axes x1y2 t "HR" lc "red"

plots/training.gnuplot

@@ -0,0 +1,23 @@
+set term qt
+set title "Subject 1066528 Sleep"
+
+# ==================== DATA =====================
+#         timestamp  heartrate  motion-avg  stage
+# 0         6.38561         52    0.002510      0
+# 1         9.38561         52    0.002611      0
+# 2        13.38564         51    0.002815      0
+# 3        18.38561         52    0.004884      0
+set multiplot
+
+plot "data/output/1066528-training-set.txt" skip 1 using 2:3 w l t "Heartrate" lw 1 lc "red"
+
+unset title
+unset xtics
+unset ytics
+unset border
+
+plot "data/output/1066528-training-set.txt" skip 1 using 2:4 w l t "Motion" lw 1 lc "grey"
+
+set y2tics ("Wake" 0, "N1" 1, "N2" 2, "N3" 3, "REM" 5)
+set y2range [-2:8]
+plot "data/output/1066528-training-set.txt" skip 1 using 2:5 w l axes x1y2 t "Stage" lw 2 lc "blue"

src/sleep_detection/main.py

@@ -20,7 +20,7 @@ from sleep_detection.sleep_epoch_classifer import SleepEpochClassifier
 SLEEP_STAGE_LABELS = [0, 1, 2]
 
 
-def normalize_sleep_stage(stage) -> float:
+def normalize_sleep_stage(stage) -> int:
     """Map raw sleep stages to the 3-class target space.
 
     Raw labels:
@@ -30,17 +30,17 @@ def normalize_sleep_stage(stage) -> float:
     Other values are treated as unknown.
     """
     if pandas.isna(stage):
-        return numpy.nan
+        return int(numpy.nan)
 
     value = int(stage)
     if value == 0:
-        return 0.0
+        return 0
     if 1 <= value <= 4:
-        return 1.0
+        return 1
     if 5 <= value <= 6:
-        return 2.0
+        return 2
 
-    return numpy.nan
+    return int(numpy.nan)
 
 
 @logger.catch
@@ -48,42 +48,7 @@ def main() -> None:
     """The Main entrypoint function."""
     cli = parse_args()
     logger.info("welcome")
-    if cli.command == "train":
+    load_session(cli.a, cli.H, cli.l)
-        sessions = load_sessions(cli.a, cli.H, cli.l)
-        epochs, features, target, label_mapping = featureize_sessions(sessions)
-
-        c = SleepEpochClassifier(
-            n_features=features.shape[1],
-            n_classes=len(SLEEP_STAGE_LABELS),
-            hidden_size=8,
-        )
-        if cli.input is not None:
-            c.load(cli.input)
-
-        logger.info(f"Using a {c.__class__.__name__}")
-
-        train(features, target, c)
-        save_network(cli.output, c, label_mapping)
-        return
-
-    if cli.command == "execute":
-        adf = load_acceleration_data(cli.a)
-        hdf = load_heartrate_data(cli.H)
-        ldf = load_labels_data(cli.l)
-
-        df = adf.join(hdf, how="outer").join(ldf, how="outer").sort_index()
-        epochs, features, target, label_mapping = featureize(df)
-
-        c = SleepEpochClassifier(
-            n_features=features.shape[1],
-            n_classes=len(SLEEP_STAGE_LABELS),
-            hidden_size=8,
-        )
-        loaded_label_mapping = c.load(cli.model)
-        logger.info(f"Using a {c.__class__.__name__}")
-        execute(epochs, features, target, c, loaded_label_mapping or label_mapping)
-        return
-
     raise ValueError(f"Unknown command: {cli.command}")
 
 
@@ -91,20 +56,10 @@ def parse_args() -> Namespace:
     parser = ArgumentParser(description="Training program.")
     parser.add_argument("-v", "--verbose", help="Increase verbosity.", action="store_true")
     parser.add_argument("-V", "--version", action="store_true", help="Show version")
-    command = parser.add_subparsers(dest="command")
 
-    train = command.add_parser("train", help="Train the model.")
+    parser.add_argument("-a", type=Path, required=True, help="Acceleration data file(s).")
-    train.add_argument("-a", type=Path, nargs="+", required=True, help="Acceleration data file(s).")
+    parser.add_argument("-H", type=Path, required=True, help="Heartrate data file(s).")
-    train.add_argument("-H", type=Path, nargs="+", required=True, help="Heartrate data file(s).")
+    parser.add_argument("-l", type=Path, required=True, help="Labels data file(s).")
-    train.add_argument("-l", type=Path, nargs="+", required=True, help="Labels data file(s).")
-    train.add_argument("-o", "--output", type=Path, default=None, required=True, help="Output .pt file.")
-    train.add_argument("-i", "--input", type=Path, default=None, required=False, help="Input .pt file.")
-
-    execute = command.add_parser("execute", help="Execute the model.")
-    execute.add_argument("-m", "--model", type=Path, default=None, required=True, help="Input .pr file to use.")
-    execute.add_argument("-a", type=Path, default=None, required=True, help="Acceleration data file.")
-    execute.add_argument("-H", type=Path, default=None, required=True, help="Heartrate data file.")
-    execute.add_argument("-l", type=Path, default=None, required=True, help="Labels data file.")
 
     return parser.parse_args()
 
@@ -181,21 +136,26 @@ def featureize_sessions(dataframes: Sequence[pandas.DataFrame]) -> tuple[pandas.
     return combined_epochs, X, y, SLEEP_STAGE_LABELS
 
 
-def load_sessions(
+def load_session(acceleration_file: Path, heartrate_file: Path, label_file: Path) -> pandas.DataFrame:
-    acceleration_files: Sequence[Path], heartrate_files: Sequence[Path], label_files: Sequence[Path]
+    """Load a session triplet."""
-) -> list[pandas.DataFrame]:
+    adf = load_acceleration_data(acceleration_file)
-    """Load multiple session triplets."""
+    hdf = load_heartrate_data(heartrate_file)
-    if not (len(acceleration_files) == len(heartrate_files) == len(label_files)):
+    ldf = load_labels_data(label_file)
-        raise ValueError("Acceleration, heartrate, and label file counts must match.")
 
-    sessions = []
+    # Average the motion into the sparser heartrate dataset.
-    for acceleration_file, heartrate_file, label_file in zip(acceleration_files, heartrate_files, label_files):
+    bin_edges = [-float("inf")] + hdf.index.tolist()
-        adf = load_acceleration_data(acceleration_file)
+    logger.info(f"({len(hdf)})")
-        hdf = load_heartrate_data(heartrate_file)
+    adf["bin"] = pandas.cut(adf.index, bins=bin_edges, labels=hdf.index, right=True)
-        ldf = load_labels_data(label_file)
+    hdf["motion-avg"] = adf.groupby("bin", observed=True)["motion"].mean()
-        sessions.append(adf.join(hdf, how="outer").join(ldf, how="outer").sort_index())
+    hdf = hdf.dropna()
 
-    return sessions
+    # Label the heartrate dataset.
+    ldf.index = ldf.index.astype(float)
+    hdf = pandas.merge_asof(hdf, ldf, on="timestamp", direction="backward")
+    with pandas.option_context('display.max_rows', None, 'display.max_columns', None):
+        print(f"{hdf}")
+    logger.info(f"\n{hdf}")
+    return hdf
 
 
 def build_epochs(data: pandas.DataFrame) -> pandas.DataFrame:
@@ -235,71 +195,34 @@ def execute(epochs: pandas.DataFrame, input: Tensor, target: Tensor, classifier:
 
 
 def load_acceleration_data(data_file: Path) -> pandas.DataFrame:
-    """Load acceleration data file.
+    """Load acceleration data file."""
-
-    Expected input format: <timestamp> <x> <y> <z>
-
-    Args:
-        data_file: The filepath for the file containing acceleration data.
-
-    Returns:
-        A pandas data frame with the loaded acceleration data.
-
-    """
     logger.info(f"Loading acceleration data file {data_file}")
     df = pandas.read_csv(data_file, sep=r"\s+", names=["timestamp", "x", "y", "z"], header=None)
-    df["timestamp"] = pandas.to_numeric(df["timestamp"], errors="coerce")
+    df = df.drop(df[df["timestamp"] < 0].index).drop_duplicates()
-    df = df.dropna(subset=["timestamp"])
-    df = df.sort_values("timestamp").set_index(pandas.to_timedelta(df["timestamp"], unit="s"))
-    df = df.drop(columns=["timestamp"])
-    df.index.name = "timestamp"
-    # df["motion"] = df[["x", "y", "z"]].pow(2).sum(axis=1).pow(0.5)
     df["motion"] = df[["x", "y", "z"]].diff().pow(2).sum(axis=1).pow(0.5)
+    df = df.set_index("timestamp")
+    logger.info(f"({len(df)})\n{df}")
     return df
 
 
 def load_heartrate_data(data_file: Path) -> pandas.DataFrame:
-    """Load heart rate data file.
+    """Load heart rate data file."""
-
-    Expected input format: <timestamp>,<heartrate>
-
-    Args:
-        data_file: The filepath for the file containing heartrate data.
-
-    Returns:
-        A pandas data frame with the loaded heartrate data.
-
-    """
     logger.info(f"Loading heartrate date file {data_file}")
-    df = pandas.read_csv(data_file, sep=",", names=["timestamp", "hr"], header=None)
+    df = pandas.read_csv(data_file, sep=",", names=["timestamp", "heartrate"], header=None)
-    df["timestamp"] = pandas.to_numeric(df["timestamp"], errors="coerce")
+    df = df.drop(df[df["timestamp"] < 0].index).drop_duplicates()
-    df = df.dropna(subset=["timestamp"])
+    df = df.set_index("timestamp")
-    df = df.sort_values("timestamp").set_index(pandas.to_timedelta(df["timestamp"], unit="s"))
+    logger.info(f"({len(df)})\n{df}")
-    df = df.drop(columns=["timestamp"])
-    df.index.name = "timestamp"
     return df
 
 
 def load_labels_data(data_file: Path) -> pandas.DataFrame:
-    """Load label data file.
+    """Load label data file."""
-
-    Expected input format: <timestamp>,<label>
-
-    Args:
-        data_file: The filepath for the file containing label data.
-
-    Returns:
-        A pandas data frame with the loaded label data.
-
-    """
     logger.info(f"Loading label date file {data_file}")
     df = pandas.read_csv(data_file, sep=r"\s+", names=["timestamp", "stage"], header=None)
-    df["timestamp"] = pandas.to_numeric(df["timestamp"], errors="coerce")
+    # df["stage"] = pandas.to_numeric(df["stage"], errors="coerce").map(normalize_sleep_stage)
-    df["stage"] = pandas.to_numeric(df["stage"], errors="coerce").map(normalize_sleep_stage)
+    df = df.drop(df[df["timestamp"] < 0].index).drop_duplicates()
-    df = df.dropna(subset=["timestamp"])
+    df = df.set_index("timestamp")
-    df = df.sort_values("timestamp").set_index(pandas.to_timedelta(df["timestamp"], unit="s"))
+    logger.info(f"({len(df)})\n{df}")
-    df = df.drop(columns=["timestamp"])
-    df.index.name = "timestamp"
     return df