初步跑通

Signed-off-by: Wenhao Ren <wenhaoren@mail.dlut.edu.cn>

Author: hangvane

Cargo.lock

@@ -669,6 +669,7 @@ mod tests {
     use super::*;
     use nydus_api::ConfigV2;
     use nydus_rafs::metadata::RafsSuperConfig;
+    use nydus_storage::backend::registry::StreamCallback;
     use nydus_storage::backend::{BackendResult, BlobReader};
     use nydus_storage::device::v5::BlobV5ChunkInfo;
     use nydus_storage::device::{BlobChunkFlags, BlobChunkInfo, BlobFeatures};
@@ -759,6 +760,16 @@ mod tests {
             Ok(i)
         }
 
+        fn try_read_stream(
+            &self,
+            _offset: u64,
+            _size: u32,
+            _processed: &mut u64,
+            _f: &mut StreamCallback,
+        ) -> BackendResult<()> {
+            unimplemented!()
+        }
+
         fn metrics(&self) -> &BackendMetrics {
             // Safe because nydusd must have backend attached with id, only image builder can no id
             // but use backend instance to upload blob.

builder/src/compact.rs

@@ -31,7 +31,8 @@ impl Blob {
         match ctx.conversion_type {
             ConversionType::DirectoryToRafs => {
                 let mut chunk_data_buf = vec![0u8; RAFS_MAX_CHUNK_SIZE as usize];
-                let (inodes, prefetch_entries) = BlobLayout::layout_blob_simple(&ctx.prefetch)?;
+                let (inodes, prefetch_entries, prefetch_idx) =
+                    BlobLayout::layout_blob_simple(&ctx.prefetch)?;
                 for (idx, node) in inodes.iter().enumerate() {
                     let mut node = node.lock().unwrap();
                     let size = node
@@ -43,6 +44,13 @@ impl Blob {
                         }
                     }
                 }
+                if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() {
+                    blob_ctx.blob_prefetch_index = prefetch_idx as u64;
+                    warn!(
+                        "CMDebug3456: blob_ctx.blob_prefetch_index = {}",
+                        blob_ctx.blob_prefetch_index
+                    );
+                }
                 Self::finalize_blob_data(ctx, blob_mgr, blob_writer)?;
             }
             ConversionType::TarToRafs

builder/src/core/blob.rs

@@ -468,6 +468,7 @@ pub struct BlobContext {
     pub blob_digester: digest::Algorithm,
     pub blob_cipher: crypt::Algorithm,
     pub blob_prefetch_size: u64,
+    pub blob_prefetch_index: u64,
     /// Whether to generate blob metadata information.
     pub blob_meta_info_enabled: bool,
     /// Data chunks stored in the data blob, for v6.
@@ -536,6 +537,7 @@ impl BlobContext {
             blob_digester: digester,
             blob_cipher: cipher,
             blob_prefetch_size: 0,
+            blob_prefetch_index: u32::MAX as u64,
             blob_meta_info_enabled: false,
             blob_meta_info,
             blob_meta_header: BlobCompressionContextHeader::default(),
@@ -698,6 +700,7 @@ impl BlobContext {
             cipher_ctx,
         );
         blob_ctx.blob_prefetch_size = blob.prefetch_size();
+        blob_ctx.blob_prefetch_index = blob.prefetch_index();
         blob_ctx.chunk_count = blob.chunk_count();
         blob_ctx.uncompressed_blob_size = blob.uncompressed_size();
         blob_ctx.compressed_blob_size = compressed_blob_size;
@@ -735,6 +738,10 @@ impl BlobContext {
 
     // TODO: check the logic to reset prefetch size
     pub fn set_blob_prefetch_size(&mut self, ctx: &BuildContext) {
+        // warn!(
+        //     "CMDebug: set prefetch size from {} to xxx, prefetch policy: {:?}",
+        //     self.blob_prefetch_size, ctx.prefetch.policy
+        // );
         if (self.uncompressed_blob_size > 0
             || (ctx.conversion_type == ConversionType::EStargzIndexToRef
                 && !self.blob_id.is_empty()))
@@ -1077,6 +1084,8 @@ impl BlobManager {
         for ctx in &self.blobs {
             let blob_id = ctx.blob_id.clone();
             let blob_prefetch_size = u32::try_from(ctx.blob_prefetch_size)?;
+            let blob_prefetch_index = u32::try_from(ctx.blob_prefetch_index)?;
+            warn!("CMDebug1111: blob_prefetch_index: {}", blob_prefetch_index);
             let chunk_count = ctx.chunk_count;
             let decompressed_blob_size = ctx.uncompressed_blob_size;
             let compressed_blob_size = ctx.compressed_blob_size;
@@ -1091,6 +1100,7 @@ impl BlobManager {
                         blob_id,
                         0,
                         blob_prefetch_size,
+                        blob_prefetch_index,
                         ctx.chunk_size,
                         chunk_count,
                         decompressed_blob_size,
@@ -1107,6 +1117,7 @@ impl BlobManager {
                         blob_id,
                         0,
                         blob_prefetch_size,
+                        blob_prefetch_index,
                         ctx.chunk_size,
                         chunk_count,
                         decompressed_blob_size,

builder/src/core/context.rs

@@ -12,8 +12,8 @@ use crate::{Overlay, Prefetch, TreeNode};
 pub struct BlobLayout {}
 
 impl BlobLayout {
-    pub fn layout_blob_simple(prefetch: &Prefetch) -> Result<(Vec<TreeNode>, usize)> {
-        let (pre, non_pre) = prefetch.get_file_nodes();
+    pub fn layout_blob_simple(prefetch: &Prefetch) -> Result<(Vec<TreeNode>, usize, usize)> {
+        let (pre, non_pre, prefetch_idx) = prefetch.get_file_nodes();
         let mut inodes: Vec<TreeNode> = pre
             .into_iter()
             .filter(|x| Self::should_dump_node(x.lock().unwrap().deref()))
@@ -27,7 +27,7 @@ impl BlobLayout {
 
         inodes.append(&mut non_prefetch_inodes);
 
-        Ok((inodes, prefetch_entries))
+        Ok((inodes, prefetch_entries, prefetch_idx))
     }
 
     #[inline]
@@ -55,7 +55,7 @@ mod tests {
         let mut prefetch = Prefetch::default();
         prefetch.insert(&tree.node, tree.node.lock().unwrap().deref());
 
-        let (inodes, prefetch_entries) = BlobLayout::layout_blob_simple(&prefetch).unwrap();
+        let (inodes, prefetch_entries, _) = BlobLayout::layout_blob_simple(&prefetch).unwrap();
         assert_eq!(inodes.len(), 1);
         assert_eq!(prefetch_entries, 0);
     }

builder/src/core/layout.rs

@@ -229,7 +229,6 @@ impl Node {
         } else {
             None
         };
-
         self.dump_node_data_with_reader(ctx, blob_mgr, blob_writer, reader.as_mut(), chunk_data_buf)
     }
 

builder/src/core/node.rs

@@ -190,13 +190,18 @@ impl Prefetch {
     /// Get node Vector of files in the prefetch list and non-prefetch list.
     /// The order of prefetch files is the same as the order of prefetch patterns.
     /// The order of non-prefetch files is the same as the order of BFS traversal of file tree.
-    pub fn get_file_nodes(&self) -> (Vec<TreeNode>, Vec<TreeNode>) {
+    pub fn get_file_nodes(&self) -> (Vec<TreeNode>, Vec<TreeNode>, usize) {
         let mut p_files = self.files_prefetch.clone();
         p_files.sort_by_key(|k| k.1);
+        let prefetch_index = if p_files.is_empty() {
+            u32::MAX as usize
+        } else {
+            p_files.first().unwrap().1 + 1
+        };
 
         let p_files = p_files.into_iter().map(|(s, _)| s).collect();
 
-        (p_files, self.files_non_prefetch.clone())
+        (p_files, self.files_non_prefetch.clone(), prefetch_index)
     }
 
     /// Get the number of ``valid`` prefetch rules.
@@ -368,7 +373,7 @@ mod tests {
         // node1, node2
         assert_eq!(prefetch.fs_prefetch_rule_count(), 2);
 
-        let (pre, non_pre) = prefetch.get_file_nodes();
+        let (pre, non_pre, _) = prefetch.get_file_nodes();
         assert_eq!(pre.len(), 4);
         assert_eq!(non_pre.len(), 1);
         let pre_str: Vec<String> = pre
@@ -384,7 +389,7 @@ mod tests {
 
         prefetch.clear();
         assert_eq!(prefetch.fs_prefetch_rule_count(), 0);
-        let (pre, non_pre) = prefetch.get_file_nodes();
+        let (pre, non_pre, _) = prefetch.get_file_nodes();
         assert_eq!(pre.len(), 0);
         assert_eq!(non_pre.len(), 0);
     }