1 #![cfg(not(loom))]
2
3 //! Asynchronous file utilities.
4 //!
5 //! This module contains utility methods for working with the file system
6 //! asynchronously. This includes reading/writing to files, and working with
7 //! directories.
8 //!
9 //! Be aware that most operating systems do not provide asynchronous file system
10 //! APIs. Because of that, Tokio will use ordinary blocking file operations
11 //! behind the scenes. This is done using the [`spawn_blocking`] threadpool to
12 //! run them in the background.
13 //!
14 //! The `tokio::fs` module should only be used for ordinary files. Trying to use
15 //! it with e.g., a named pipe on Linux can result in surprising behavior,
16 //! such as hangs during runtime shutdown. For special files, you should use a
17 //! dedicated type such as [`tokio::net::unix::pipe`] or [`AsyncFd`] instead.
18 //!
19 //! Currently, Tokio will always use [`spawn_blocking`] on all platforms, but it
20 //! may be changed to use asynchronous file system APIs such as io_uring in the
21 //! future.
22 //!
23 //! # Usage
24 //!
25 //! The easiest way to use this module is to use the utility functions that
26 //! operate on entire files:
27 //!
28 //! * [`tokio::fs::read`](fn@crate::fs::read)
29 //! * [`tokio::fs::read_to_string`](fn@crate::fs::read_to_string)
30 //! * [`tokio::fs::write`](fn@crate::fs::write)
31 //!
32 //! The two `read` functions reads the entire file and returns its contents.
33 //! The `write` function takes the contents of the file and writes those
34 //! contents to the file. It overwrites the existing file, if any.
35 //!
36 //! For example, to read the file:
37 //!
38 //! ```
39 //! # async fn dox() -> std::io::Result<()> {
40 //! let contents = tokio::fs::read_to_string("my_file.txt").await?;
41 //!
42 //! println!("File has {} lines.", contents.lines().count());
43 //! # Ok(())
44 //! # }
45 //! ```
46 //!
47 //! To overwrite the file:
48 //!
49 //! ```
50 //! # async fn dox() -> std::io::Result<()> {
51 //! let contents = "First line.\nSecond line.\nThird line.\n";
52 //!
53 //! tokio::fs::write("my_file.txt", contents.as_bytes()).await?;
54 //! # Ok(())
55 //! # }
56 //! ```
57 //!
58 //! ## Using `File`
59 //!
60 //! The main type for interacting with files is [`File`]. It can be used to read
61 //! from and write to a given file. This is done using the [`AsyncRead`] and
62 //! [`AsyncWrite`] traits. This type is generally used when you want to do
63 //! something more complex than just reading or writing the entire contents in
64 //! one go.
65 //!
66 //! **Note:** It is important to use [`flush`] when writing to a Tokio
67 //! [`File`]. This is because calls to `write` will return before the write has
68 //! finished, and [`flush`] will wait for the write to finish. (The write will
69 //! happen even if you don't flush; it will just happen later.) This is
70 //! different from [`std::fs::File`], and is due to the fact that `File` uses
71 //! `spawn_blocking` behind the scenes.
72 //!
73 //! For example, to count the number of lines in a file without loading the
74 //! entire file into memory:
75 //!
76 //! ```no_run
77 //! use tokio::fs::File;
78 //! use tokio::io::AsyncReadExt;
79 //!
80 //! # async fn dox() -> std::io::Result<()> {
81 //! let mut file = File::open("my_file.txt").await?;
82 //!
83 //! let mut chunk = vec![0; 4096];
84 //! let mut number_of_lines = 0;
85 //! loop {
86 //! let len = file.read(&mut chunk).await?;
87 //! if len == 0 {
88 //! // Length of zero means end of file.
89 //! break;
90 //! }
91 //! for &b in &chunk[..len] {
92 //! if b == b'\n' {
93 //! number_of_lines += 1;
94 //! }
95 //! }
96 //! }
97 //!
98 //! println!("File has {} lines.", number_of_lines);
99 //! # Ok(())
100 //! # }
101 //! ```
102 //!
103 //! For example, to write a file line-by-line:
104 //!
105 //! ```no_run
106 //! use tokio::fs::File;
107 //! use tokio::io::AsyncWriteExt;
108 //!
109 //! # async fn dox() -> std::io::Result<()> {
110 //! let mut file = File::create("my_file.txt").await?;
111 //!
112 //! file.write_all(b"First line.\n").await?;
113 //! file.write_all(b"Second line.\n").await?;
114 //! file.write_all(b"Third line.\n").await?;
115 //!
116 //! // Remember to call `flush` after writing!
117 //! file.flush().await?;
118 //! # Ok(())
119 //! # }
120 //! ```
121 //!
122 //! ## Tuning your file IO
123 //!
124 //! Tokio's file uses [`spawn_blocking`] behind the scenes, and this has serious
125 //! performance consequences. To get good performance with file IO on Tokio, it
126 //! is recommended to batch your operations into as few `spawn_blocking` calls
127 //! as possible.
128 //!
129 //! One example of this difference can be seen by comparing the two reading
130 //! examples above. The first example uses [`tokio::fs::read`], which reads the
131 //! entire file in a single `spawn_blocking` call, and then returns it. The
132 //! second example will read the file in chunks using many `spawn_blocking`
133 //! calls. This means that the second example will most likely be more expensive
134 //! for large files. (Of course, using chunks may be necessary for very large
135 //! files that don't fit in memory.)
136 //!
137 //! The following examples will show some strategies for this:
138 //!
139 //! When creating a file, write the data to a `String` or `Vec<u8>` and then
140 //! write the entire file in a single `spawn_blocking` call with
141 //! `tokio::fs::write`.
142 //!
143 //! ```no_run
144 //! # async fn dox() -> std::io::Result<()> {
145 //! let mut contents = String::new();
146 //!
147 //! contents.push_str("First line.\n");
148 //! contents.push_str("Second line.\n");
149 //! contents.push_str("Third line.\n");
150 //!
151 //! tokio::fs::write("my_file.txt", contents.as_bytes()).await?;
152 //! # Ok(())
153 //! # }
154 //! ```
155 //!
156 //! Use [`BufReader`] and [`BufWriter`] to buffer many small reads or writes
157 //! into a few large ones. This example will most likely only perform one
158 //! `spawn_blocking` call.
159 //!
160 //! ```no_run
161 //! use tokio::fs::File;
162 //! use tokio::io::{AsyncWriteExt, BufWriter};
163 //!
164 //! # async fn dox() -> std::io::Result<()> {
165 //! let mut file = BufWriter::new(File::create("my_file.txt").await?);
166 //!
167 //! file.write_all(b"First line.\n").await?;
168 //! file.write_all(b"Second line.\n").await?;
169 //! file.write_all(b"Third line.\n").await?;
170 //!
171 //! // Due to the BufWriter, the actual write and spawn_blocking
172 //! // call happens when you flush.
173 //! file.flush().await?;
174 //! # Ok(())
175 //! # }
176 //! ```
177 //!
178 //! Manually use [`std::fs`] inside [`spawn_blocking`].
179 //!
180 //! ```no_run
181 //! use std::fs::File;
182 //! use std::io::{self, Write};
183 //! use tokio::task::spawn_blocking;
184 //!
185 //! # async fn dox() -> std::io::Result<()> {
186 //! spawn_blocking(move || {
187 //! let mut file = File::create("my_file.txt")?;
188 //!
189 //! file.write_all(b"First line.\n")?;
190 //! file.write_all(b"Second line.\n")?;
191 //! file.write_all(b"Third line.\n")?;
192 //!
193 //! // Unlike Tokio's file, the std::fs file does
194 //! // not need flush.
195 //!
196 //! io::Result::Ok(())
197 //! }).await.unwrap()?;
198 //! # Ok(())
199 //! # }
200 //! ```
201 //!
202 //! It's also good to be aware of [`File::set_max_buf_size`], which controls the
203 //! maximum amount of bytes that Tokio's [`File`] will read or write in a single
204 //! [`spawn_blocking`] call. The default is two megabytes, but this is subject
205 //! to change.
206 //!
207 //! [`spawn_blocking`]: fn@crate::task::spawn_blocking
208 //! [`AsyncRead`]: trait@crate::io::AsyncRead
209 //! [`AsyncWrite`]: trait@crate::io::AsyncWrite
210 //! [`BufReader`]: struct@crate::io::BufReader
211 //! [`BufWriter`]: struct@crate::io::BufWriter
212 //! [`tokio::net::unix::pipe`]: crate::net::unix::pipe
213 //! [`AsyncFd`]: crate::io::unix::AsyncFd
214 //! [`flush`]: crate::io::AsyncWriteExt::flush
215 //! [`tokio::fs::read`]: fn@crate::fs::read
216
217 mod canonicalize;
218 pub use self::canonicalize::canonicalize;
219
220 mod create_dir;
221 pub use self::create_dir::create_dir;
222
223 mod create_dir_all;
224 pub use self::create_dir_all::create_dir_all;
225
226 mod dir_builder;
227 pub use self::dir_builder::DirBuilder;
228
229 mod file;
230 pub use self::file::File;
231
232 mod hard_link;
233 pub use self::hard_link::hard_link;
234
235 mod metadata;
236 pub use self::metadata::metadata;
237
238 mod open_options;
239 pub use self::open_options::OpenOptions;
240
241 mod read;
242 pub use self::read::read;
243
244 mod read_dir;
245 pub use self::read_dir::{read_dir, DirEntry, ReadDir};
246
247 mod read_link;
248 pub use self::read_link::read_link;
249
250 mod read_to_string;
251 pub use self::read_to_string::read_to_string;
252
253 mod remove_dir;
254 pub use self::remove_dir::remove_dir;
255
256 mod remove_dir_all;
257 pub use self::remove_dir_all::remove_dir_all;
258
259 mod remove_file;
260 pub use self::remove_file::remove_file;
261
262 mod rename;
263 pub use self::rename::rename;
264
265 mod set_permissions;
266 pub use self::set_permissions::set_permissions;
267
268 mod symlink_metadata;
269 pub use self::symlink_metadata::symlink_metadata;
270
271 mod write;
272 pub use self::write::write;
273
274 mod copy;
275 pub use self::copy::copy;
276
277 mod try_exists;
278 pub use self::try_exists::try_exists;
279
280 #[cfg(test)]
281 mod mocks;
282
283 feature! {
284 #![unix]
285
286 mod symlink;
287 pub use self::symlink::symlink;
288 }
289
290 cfg_windows! {
291 mod symlink_dir;
292 pub use self::symlink_dir::symlink_dir;
293
294 mod symlink_file;
295 pub use self::symlink_file::symlink_file;
296 }
297
298 use std::io;
299
300 #[cfg(not(test))]
301 use crate::blocking::spawn_blocking;
302 #[cfg(test)]
303 use mocks::spawn_blocking;
304
asyncify<F, T>(f: F) -> io::Result<T> where F: FnOnce() -> io::Result<T> + Send + 'static, T: Send + 'static,305 pub(crate) async fn asyncify<F, T>(f: F) -> io::Result<T>
306 where
307 F: FnOnce() -> io::Result<T> + Send + 'static,
308 T: Send + 'static,
309 {
310 match spawn_blocking(f).await {
311 Ok(res) => res,
312 Err(_) => Err(io::Error::new(
313 io::ErrorKind::Other,
314 "background task failed",
315 )),
316 }
317 }
318