Merge branch 'main' into jan/migrate-grpc

crates/build/re_types_builder/src/codegen/cpp/mod.rs

@@ -1831,8 +1831,7 @@ fn archetype_serialize(type_ident: &Ident, obj: &Object, hpp_includes: &mut Incl
             #NEWLINE_TOKEN
             #(#push_batches)*
             {
-                auto indicator = #type_ident::IndicatorComponent();
-                auto result = ComponentBatch::from_loggable(indicator);
+                auto result = ComponentBatch::from_indicator<#type_ident>();
                 RR_RETURN_NOT_OK(result.error);
                 cells.emplace_back(std::move(result.value));
             }

crates/build/re_types_builder/src/codegen/python/mod.rs

@@ -2645,15 +2645,14 @@ fn quote_columnar_methods(reporter: &Reporter, obj: &Object, objects: &Objects)
                     {init_args},
                 )
 
-            batches = [batch for batch in inst.as_component_batches() if isinstance(batch, DescribedComponentBatch)]
+            batches = inst.as_component_batches(include_indicators=False)
             if len(batches) == 0:
                 return ComponentColumnList([])
 
             lengths = np.ones(len(batches[0]._batch.as_arrow_array()))
             columns = [batch.partition(lengths) for batch in batches]
 
-            indicator_batch = DescribedComponentBatch(cls.indicator(), cls.indicator().component_descriptor())
-            indicator_column = indicator_batch.partition(np.zeros(len(lengths)))
+            indicator_column = cls.indicator().partition(np.zeros(len(lengths)))
 
             return ComponentColumnList([indicator_column] + columns)
         "#

crates/store/re_types_core/src/archetype.rs

@@ -50,7 +50,6 @@ pub trait Archetype {
     /// Creates a [`ComponentBatch`] out of the associated [`Self::Indicator`] component.
     ///
     /// This allows for associating arbitrary indicator components with arbitrary data.
-    /// Check out the `manual_indicator` API example to see what's possible.
     fn indicator() -> SerializedComponentBatch;
 
     /// Returns all component descriptors that _must_ be provided by the user when constructing this archetype.

crates/top/rerun/src/commands/rrd/compare.rs

@@ -6,13 +6,24 @@ use std::{
 use anyhow::Context as _;
 use itertools::{izip, Itertools};
 
+use re_chunk::Chunk;
+
 // ---
 
 #[derive(Debug, Clone, clap::Parser)]
 pub struct CompareCommand {
     path_to_rrd1: String,
     path_to_rrd2: String,
 
+    /// If specified, the comparison will focus purely on semantics, ignoring order.
+    ///
+    /// The Rerun data model is itself unordered, and because many of the internal pipelines are
+    /// asynchronous by nature, it is very easy to end up with semantically identical, but
+    /// differently ordered data.
+    /// In most cases, the distinction is irrelevant, and you'd rather the comparison succeeds.
+    #[clap(long, default_value_t = false)]
+    unordered: bool,
+
     /// If specified, dumps both .rrd files as tables.
     #[clap(long, default_value_t = false)]
     full_dump: bool,
@@ -27,6 +38,7 @@ impl CompareCommand {
         let Self {
             path_to_rrd1,
             path_to_rrd2,
+            unordered,
             full_dump,
         } = self;
 
@@ -64,17 +76,36 @@ impl CompareCommand {
             re_format::format_uint(chunks2.len()),
         );
 
-        for (chunk1, chunk2) in izip!(chunks1, chunks2) {
-            anyhow::ensure!(
-                re_chunk::Chunk::are_similar(&chunk1, &chunk2),
-                "Chunks do not match:\n{}",
-                similar_asserts::SimpleDiff::from_str(
-                    &format!("{chunk1}"),
-                    &format!("{chunk2}"),
-                    "got",
-                    "expected",
-                ),
-            );
+        let mut unordered_failed = false;
+        if *unordered {
+            let mut chunks2_opt: Vec<Option<Arc<Chunk>>> =
+                chunks2.clone().into_iter().map(Some).collect_vec();
+            'outer: for chunk1 in &chunks1 {
+                for chunk2 in chunks2_opt.iter_mut().filter(|c| c.is_some()) {
+                    #[allow(clippy::unwrap_used)]
+                    if re_chunk::Chunk::are_similar(chunk1, chunk2.as_ref().unwrap()) {
+                        *chunk2 = None;
+                        continue 'outer;
+                    }
+                }
+                unordered_failed = true;
+                break;
+            }
+        }
+
+        if !*unordered || unordered_failed {
+            for (chunk1, chunk2) in izip!(chunks1, chunks2) {
+                anyhow::ensure!(
+                    re_chunk::Chunk::are_similar(&chunk1, &chunk2),
+                    "Chunks do not match:\n{}",
+                    similar_asserts::SimpleDiff::from_str(
+                        &format!("{chunk1}"),
+                        &format!("{chunk2}"),
+                        "got",
+                        "expected",
+                    ),
+                );
+            }
         }
 
         re_log::debug!("{path_to_rrd1:?} and {path_to_rrd2:?} are similar enough.");

docs/content/howto/logging/custom-data.md

@@ -37,7 +37,7 @@ class LabeledPoints:
     points: np.ndarray
     labels: List[str]
 
-    def as_component_batches(self) -> Iterable[rr.ComponentBatch]:
+    def as_component_batches(self) -> list[rr.ComponentBatch]:
         return rr.Points3D(positions=self.points,
                            labels=self.labels).as_component_batches()
 …

docs/snippets/all/archetypes/image_send_columns.cpp

@@ -32,7 +32,7 @@ int main() {
         rerun::ColorModel::RGB,
         rerun::ChannelDatatype::U8
     );
-    rec.log_static("images", rerun::borrow(&format), rerun::Image::IndicatorComponent());
+    rec.log_static("images", rerun::Image::update_fields().with_format(format));
 
     // Split up the image data into several components referencing the underlying data.
     const size_t image_size_in_bytes = width * height * 3;

docs/snippets/all/archetypes/image_send_columns.py

@@ -16,8 +16,8 @@
     images[t, 50:150, (t * 10) : (t * 10 + 100), 1] = 255
 
 # Log the ImageFormat and indicator once, as static.
-format_static = rr.components.ImageFormat(width=width, height=height, color_model="RGB", channel_datatype="U8")
-rr.send_columns("images", indexes=[], columns=rr.Image.columns(format=format_static))
+format = rr.components.ImageFormat(width=width, height=height, color_model="RGB", channel_datatype="U8")
+rr.log("images", rr.Image.from_fields(format=format), static=True)
 
 # Send all images at once.
 rr.send_columns(

docs/snippets/all/archetypes/image_send_columns.rs

@@ -20,15 +20,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             .fill(255);
     }
 
-    // Send the ImageFormat and indicator once, as static.
+    // Log the ImageFormat and indicator once, as static.
     let format = rerun::components::ImageFormat::rgb8([width as _, height as _]);
-    rec.send_columns(
-        "images",
-        [],
-        rerun::Image::update_fields()
-            .with_format(format)
-            .columns_of_unit_batches()?,
-    )?;
+    rec.log_static("images", &rerun::Image::update_fields().with_format(format))?;
 
     // Split up the image data into several components referencing the underlying data.
     let image_size_in_bytes = width * height * 3;

docs/snippets/all/archetypes/mesh3d_partial_updates.cpp

@@ -43,6 +43,9 @@ int main() {
             mul_pos(factor, vertex_positions[1]),
             mul_pos(factor, vertex_positions[2]),
         };
-        rec.log("triangle", modified_vertex_positions);
+        rec.log(
+            "triangle",
+            rerun::Mesh3D::update_fields().with_vertex_positions(modified_vertex_positions)
+        );
     }
 }

docs/snippets/all/descriptors/descr_custom_archetype.cpp

@@ -26,9 +26,6 @@ struct rerun::Loggable<CustomPosition3D> {
 
 /// A custom archetype that extends Rerun's builtin `rerun::Points3D` archetype with a custom component.
 struct CustomPoints3D {
-    static constexpr const char IndicatorComponentName[] = "user.CustomPoints3DIndicator";
-    using IndicatorComponent = rerun::components::IndicatorComponent<IndicatorComponentName>;
-
     rerun::Collection<CustomPosition3D> positions;
     std::optional<rerun::Collection<rerun::Color>> colors;
 };
@@ -38,9 +35,6 @@ struct rerun::AsComponents<CustomPoints3D> {
     static Result<std::vector<ComponentBatch>> serialize(const CustomPoints3D& archetype) {
         std::vector<rerun::ComponentBatch> batches;
 
-        CustomPoints3D::IndicatorComponent indicator;
-        batches.push_back(ComponentBatch::from_loggable(indicator).value_or_throw());
-
         auto positions_descr = rerun::Loggable<CustomPosition3D>::Descriptor
                                    .or_with_archetype_name("user.CustomPoints3D")
                                    .or_with_archetype_field_name("custom_positions");

docs/snippets/all/descriptors/descr_custom_archetype.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Any, Iterable
+from typing import Any
 
 import numpy.typing as npt
 import rerun as rr  # pip install rerun-sdk
@@ -22,8 +22,8 @@ def __init__(self: Any, positions: npt.ArrayLike, colors: npt.ArrayLike) -> None
             archetype_field_name="colors",
         )
 
-    def as_component_batches(self) -> Iterable[rr.ComponentBatchLike]:
-        return [rr.IndicatorComponentBatch("user.CustomPoints3D"), self.positions, self.colors]
+    def as_component_batches(self) -> list[rr.DescribedComponentBatch]:
+        return [self.positions, self.colors]
 
 
 rr.init("rerun_example_descriptors_custom_archetype")

docs/snippets/all/descriptors/descr_custom_archetype.rs

@@ -6,10 +6,6 @@ struct CustomPoints3D {
 }
 
 impl CustomPoints3D {
-    fn indicator() -> rerun::NamedIndicatorComponent {
-        rerun::NamedIndicatorComponent("user.CustomPoints3DIndicator".into())
-    }
-
     fn overridden_position_descriptor() -> ComponentDescriptor {
         ComponentDescriptor {
             archetype_name: Some("user.CustomPoints3D".into()),
@@ -28,7 +24,6 @@ impl CustomPoints3D {
 impl rerun::AsComponents for CustomPoints3D {
     fn as_serialized_batches(&self) -> Vec<rerun::SerializedComponentBatch> {
         [
-            Self::indicator().serialized(),
             self.positions.serialized().map(|positions| {
                 positions.with_descriptor_override(Self::overridden_position_descriptor())
             }),
@@ -105,11 +100,6 @@ fn check_tags(rec: &rerun::RecordingStream) {
         descriptors.sort();
 
         let expected = vec![
-            ComponentDescriptor {
-                archetype_name: None,
-                archetype_field_name: None,
-                component_name: "user.CustomPoints3DIndicator".into(),
-            },
             ComponentDescriptor {
                 archetype_name: Some("user.CustomPoints3D".into()),
                 archetype_field_name: Some("colors".into()),

docs/snippets/all/tutorials/custom_data.cpp

@@ -29,9 +29,6 @@ struct rerun::Loggable<Confidence> {
 
 /// A custom archetype that extends Rerun's builtin `rerun::Points3D` archetype with a custom component.
 struct CustomPoints3D {
-    static constexpr const char IndicatorComponentName[] = "user.CustomPoints3DIndicator";
-    using IndicatorComponent = rerun::components::IndicatorComponent<IndicatorComponentName>;
-
     rerun::Points3D points;
     // Using a rerun::Collection is not strictly necessary, you could also use an std::vector for example,
     // but useful for avoiding allocations since `rerun::Collection` can borrow data from other containers.
@@ -43,10 +40,6 @@ struct rerun::AsComponents<CustomPoints3D> {
     static Result<std::vector<ComponentBatch>> serialize(const CustomPoints3D& archetype) {
         auto batches = AsComponents<rerun::Points3D>::serialize(archetype.points).value_or_throw();
 
-        // Add a custom indicator component.
-        CustomPoints3D::IndicatorComponent indicator;
-        batches.push_back(ComponentBatch::from_loggable(indicator).value_or_throw());
-
         // Add custom confidence components if present.
         if (archetype.confidences) {
             auto descriptor = rerun::Loggable<Confidence>::Descriptor

docs/snippets/all/tutorials/custom_data.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import argparse
-from typing import Any, Iterable
+from typing import Any
 
 import numpy as np
 import numpy.typing as npt
@@ -35,10 +35,9 @@ def __init__(self: Any, positions: npt.ArrayLike, confidences: npt.ArrayLike) ->
             archetype_name="user.CustomPoints3D", archetype_field_name="confidences"
         )
 
-    def as_component_batches(self) -> Iterable[rr.ComponentBatchLike]:
+    def as_component_batches(self) -> list[rr.DescribedComponentBatch]:
         return (
             list(self.points3d.as_component_batches())  # The components from Points3D
-            + [rr.IndicatorComponentBatch("user.CustomPoints3D")]  # Our custom indicator
             + [self.confidences]  # Custom confidence data
         )
 

docs/snippets/all/tutorials/custom_data.rs

@@ -19,22 +19,19 @@ struct CustomPoints3D {
 
 impl rerun::AsComponents for CustomPoints3D {
     fn as_serialized_batches(&self) -> Vec<SerializedComponentBatch> {
-        let indicator = rerun::NamedIndicatorComponent("user.CustomPoints3DIndicator".into());
         self.points3d
             .as_serialized_batches()
             .into_iter()
             .chain(
-                [
-                    indicator.serialized(),
-                    self.confidences
-                        .as_ref()
-                        .and_then(|batch| batch.serialized())
-                        .map(|batch|
+                [self
+                    .confidences
+                    .as_ref()
+                    .and_then(|batch| batch.serialized())
+                    .map(|batch|
                             // Optionally override the descriptor with extra information.
                             batch
                                 .or_with_archetype_name(|| "user.CustomPoints3D".into())
-                                .or_with_archetype_field_name(|| "confidences".into())),
-                ]
+                                .or_with_archetype_field_name(|| "confidences".into()))]
                 .into_iter()
                 .flatten(),
             )

docs/snippets/snippets.toml

@@ -2,7 +2,6 @@
 
 # These snippets will be excluded from the snippet index, unconditionally.
 [snippets_ref.snippets.opt_out]
-"archetypes/manual_indicator" = []
 
 # These archetypes will ignore the associated snippets in the snippet index.
 [snippets_ref.archetypes.opt_out]
@@ -198,15 +197,8 @@ quick_start = [ # These examples don't have exactly the same implementation.
   "py",
   "rust",
 ]
-"archetypes/arrows3d_simple" = [ # TODO(#3206): examples use different RNGs
-  "cpp",
+"archetypes/arrows3d_simple" = [ # Python uses doubles
   "py",
-  "rust",
-]
-"archetypes/asset3d_out_of_tree" = [ # float issues since calculation is done slightly differently (also, Python uses doubles)
-  "cpp",
-  "py",
-  "rust",
 ]
 "archetypes/bar_chart" = [ # On Windows this logs f64 instead of u64 unless a numpy array with explicit type is used.
   "py",
@@ -221,11 +213,9 @@ quick_start = [ # These examples don't have exactly the same implementation.
   "py",
   "rust",
 ]
-"archetypes/mesh3d_partial_updates" = [ # TODO(andreas): use new partial update api
-  "cpp",
+"archetypes/mesh3d_partial_updates" = [ # Python uses doubles
   "py",
-  "rust",
-] # float precision issues
+]
 "archetypes/pinhole_simple" = [ # TODO(#3206): examples use different RNGs
   "cpp",
   "py",
@@ -249,17 +239,7 @@ quick_start = [ # These examples don't have exactly the same implementation.
   "py",
   "rust",
 ]
-"archetypes/series_point_style" = [ # TODO(#5116): trigonometric functions have slightly different outcomes
-  "cpp",
-  "py",
-  "rust",
-]
-"archetypes/series_line_style" = [ # TODO(#5116):trigonometric functions have slightly different outcomes
-  "cpp",
-  "py",
-  "rust",
-]
-"archetypes/text_log_integration" = [ # The entity path will differ because the Rust code is part of a library
+"archetypes/text_log_integration" = [ # The entity path will differ because the integrations work differently
   "cpp",
   "py",
   "rust",
@@ -277,19 +257,8 @@ quick_start = [ # These examples don't have exactly the same implementation.
   "py",
   "rust",
 ]
-"archetypes/image_send_columns" = [ # This mixes `log` and `send_columns`. Since `log` is suspect to delays by the batcher, this test gets flaky.
-  "cpp",
-  "rust",
-]
-"archetypes/mesh3d_instancing" = [ # TODO(#3235): Slight floating point differences in deg to rad conversion.
-  "cpp",
-  "py",
-  "rust",
-]
-"archetypes/video_auto_frames" = [ # This mixes `log` and `send_columns`. Since `log` is suspect to delays by the batcher, this test gets flaky.
-  "cpp",
+"archetypes/mesh3d_instancing" = [ # Python uses doubles
   "py",
-  "rust",
 ]
 
 # `$config_dir` will be replaced with the absolute path of `docs/snippets`.