diff --git a/littlefs/.travis.yml b/littlefs/.travis.yml new file mode 100644 index 0000000000..0936b29f44 --- /dev/null +++ b/littlefs/.travis.yml @@ -0,0 +1,4 @@ +script: + - CFLAGS="-DLFS_READ_SIZE=16 -DLFS_PROG_SIZE=16" make test + - CFLAGS="-DLFS_READ_SIZE=1 -DLFS_PROG_SIZE=1" make test + - CFLAGS="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512" make test diff --git a/littlefs/DESIGN.md b/littlefs/DESIGN.md new file mode 100644 index 0000000000..3381c7919c --- /dev/null +++ b/littlefs/DESIGN.md @@ -0,0 +1,973 @@ +## The design of the little filesystem + +The littlefs is a little fail-safe filesystem designed for embedded systems. + +``` + | | | .---._____ + .-----. | | +--|o |---| littlefs | +--| |---| | + '-----' '----------' + | | | +``` + +For a bit of backstory, the littlefs was developed with the goal of learning +more about filesystem design by tackling the relative unsolved problem of +managing a robust filesystem resilient to power loss on devices +with limited RAM and ROM. + +The embedded systems the littlefs is targeting are usually 32bit +microcontrollers with around 32Kbytes of RAM and 512Kbytes of ROM. These are +often paired with SPI NOR flash chips with about 4Mbytes of flash storage. + +Flash itself is a very interesting piece of technology with quite a bit of +nuance. Unlike most other forms of storage, writing to flash requires two +operations: erasing and programming. The programming operation is relatively +cheap, and can be very granular. For NOR flash specifically, byte-level +programs are quite common. Erasing, however, requires an expensive operation +that forces the state of large blocks of memory to reset in a destructive +reaction that gives flash its name. The [Wikipedia entry](https://en.wikipedia.org/wiki/Flash_memory) +has more information if you are interesting in how this works. + +This leaves us with an interesting set of limitations that can be simplified +to three strong requirements: + +1. **Fail-safe** - This is actually the main goal of the littlefs and the focus + of this project. Embedded systems are usually designed without a shutdown + routine and a notable lack of user interface for recovery, so filesystems + targeting embedded systems should be prepared to lose power an any given + time. + + Despite this state of things, there are very few embedded filesystems that + handle power loss in a reasonable manner, and can become corrupted if the + user is unlucky enough. + +2. **Wear awareness** - Due to the destructive nature of flash, most flash + chips have a limited number of erase cycles, usually in the order of around + 100,000 erases per block for NOR flash. Filesystems that don't take wear + into account can easily burn through blocks used to store frequently updated + metadata. + + Consider the [FAT filesystem](https://en.wikipedia.org/wiki/Design_of_the_FAT_file_system), + which stores a file allocation table (FAT) at a specific offset from the + beginning of disk. Every block allocation will update this table, and after + 100,000 updates, the block will likely go bad, rendering the filesystem + unusable even if there are many more erase cycles available on the storage. + +3. **Bounded RAM/ROM** - Even with the design difficulties presented by the + previous two limitations, we have already seen several flash filesystems + developed on PCs that handle power loss just fine, such as the + logging filesystems. However, these filesystems take advantage of the + relatively cheap access to RAM, and use some rather... opportunistic... + techniques, such as reconstructing the entire directory structure in RAM. + These operations make perfect sense when the filesystem's only concern is + erase cycles, but the idea is a bit silly on embedded systems. + + To cater to embedded systems, the littlefs has the simple limitation of + using only a bounded amount of RAM and ROM. That is, no matter what is + written to the filesystem, and no matter how large the underlying storage + is, the littlefs will always use the same amount of RAM and ROM. This + presents a very unique challenge, and makes presumably simple operations, + such as iterating through the directory tree, surprisingly difficult. + +## Existing designs? + +There are of course, many different existing filesystem. Heres a very rough +summary of the general ideas behind some of them. + +Most of the existing filesystems fall into the one big category of filesystem +designed in the early days of spinny magnet disks. While there is a vast amount +of interesting technology and ideas in this area, the nature of spinny magnet +disks encourage properties such as grouping writes near each other, that don't +make as much sense on recent storage types. For instance, on flash, write +locality is not as important and can actually increase wear destructively. + +One of the most popular designs for flash filesystems is called the +[logging filesystem](https://en.wikipedia.org/wiki/Log-structured_file_system). +The flash filesystems [jffs](https://en.wikipedia.org/wiki/JFFS) +and [yaffs](https://en.wikipedia.org/wiki/YAFFS) are good examples. In +logging filesystem, data is not store in a data structure on disk, but instead +the changes to the files are stored on disk. This has several neat advantages, +such as the fact that the data is written in a cyclic log format naturally +wear levels as a side effect. And, with a bit of error detection, the entire +filesystem can easily be designed to be resilient to power loss. The +journalling component of most modern day filesystems is actually a reduced +form of a logging filesystem. However, logging filesystems have a difficulty +scaling as the size of storage increases. And most filesystems compensate by +caching large parts of the filesystem in RAM, a strategy that is unavailable +for embedded systems. + +Another interesting filesystem design technique that the littlefs borrows the +most from, is the [copy-on-write (COW)](https://en.wikipedia.org/wiki/Copy-on-write). +A good example of this is the [btrfs](https://en.wikipedia.org/wiki/Btrfs) +filesystem. COW filesystems can easily recover from corrupted blocks and have +natural protection against power loss. However, if they are not designed with +wear in mind, a COW filesystem could unintentionally wear down the root block +where the COW data structures are synchronized. + +## Metadata pairs + +The core piece of technology that provides the backbone for the littlefs is +the concept of metadata pairs. The key idea here, is that any metadata that +needs to be updated atomically is stored on a pair of blocks tagged with +a revision count and checksum. Every update alternates between these two +pairs, so that at any time there is always a backup containing the previous +state of the metadata. + +Consider a small example where each metadata pair has a revision count, +a number as data, and the xor of the block as a quick checksum. If +we update the data to a value of 9, and then to a value of 5, here is +what the pair of blocks may look like after each update: +``` + block 1 block 2 block 1 block 2 block 1 block 2 +.---------.---------. .---------.---------. .---------.---------. +| rev: 1 | rev: 0 | | rev: 1 | rev: 2 | | rev: 3 | rev: 2 | +| data: 3 | data: 0 | -> | data: 3 | data: 9 | -> | data: 5 | data: 9 | +| xor: 2 | xor: 0 | | xor: 2 | xor: 11 | | xor: 6 | xor: 11 | +'---------'---------' '---------'---------' '---------'---------' + let data = 9 let data = 5 +``` + +After each update, we can find the most up to date value of data by looking +at the revision count. + +Now consider what the blocks may look like if we suddenly loss power while +changing the value of data to 5: +``` + block 1 block 2 block 1 block 2 block 1 block 2 +.---------.---------. .---------.---------. .---------.---------. +| rev: 1 | rev: 0 | | rev: 1 | rev: 2 | | rev: 3 | rev: 2 | +| data: 3 | data: 0 | -> | data: 3 | data: 9 | -x | data: 3 | data: 9 | +| xor: 2 | xor: 0 | | xor: 2 | xor: 11 | | xor: 2 | xor: 11 | +'---------'---------' '---------'---------' '---------'---------' + let data = 9 let data = 5 + powerloss!!! +``` + +In this case, block 1 was partially written with a new revision count, but +the littlefs hadn't made it to updating the value of data. However, if we +check our checksum we notice that block 1 was corrupted. So we fall back to +block 2 and use the value 9. + +Using this concept, the littlefs is able to update metadata blocks atomically. +There are a few other tweaks, such as using a 32bit crc and using sequence +arithmetic to handle revision count overflow, but the basic concept +is the same. These metadata pairs define the backbone of the littlefs, and the +rest of the filesystem is built on top of these atomic updates. + +## Files + +Now, the metadata pairs do come with some drawbacks. Most notably, each pair +requires two blocks for each block of data. I'm sure users would be very +unhappy if their storage was suddenly cut in half! Instead of storing +everything in these metadata blocks, the littlefs uses a COW data structure +for files which is in turn pointed to by a metadata block. When +we update a file, we create a copies of any blocks that are modified until +the metadata blocks are updated with the new copy. Once the metadata block +points to the new copy, we deallocate the old blocks that are no longer in use. + +Here is what updating a one-block file may look like: +``` + block 1 block 2 block 1 block 2 block 1 block 2 +.---------.---------. .---------.---------. .---------.---------. +| rev: 1 | rev: 0 | | rev: 1 | rev: 0 | | rev: 1 | rev: 2 | +| file: 4 | file: 0 | -> | file: 4 | file: 0 | -> | file: 4 | file: 5 | +| xor: 5 | xor: 0 | | xor: 5 | xor: 0 | | xor: 5 | xor: 7 | +'---------'---------' '---------'---------' '---------'---------' + | | | + v v v + block 4 block 4 block 5 block 4 block 5 +.--------. .--------. .--------. .--------. .--------. +| old | | old | | new | | old | | new | +| data | | data | | data | | data | | data | +| | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' + update data in file update metadata pair +``` + +It doesn't matter if we lose power while writing block 5 with the new data, +since the old data remains unmodified in block 4. This example also +highlights how the atomic updates of the metadata blocks provide a +synchronization barrier for the rest of the littlefs. + +At this point, it may look like we are wasting an awfully large amount +of space on the metadata. Just looking at that example, we are using +three blocks to represent a file that fits comfortably in one! So instead +of giving each file a metadata pair, we actually store the metadata for +all files contained in a single directory in a single metadata block. + +Now we could just leave files here, copying the entire file on write +provides the synchronization without the duplicated memory requirements +of the metadata blocks. However, we can do a bit better. + +## CTZ linked-lists + +There are many different data structures for representing the actual +files in filesystems. Of these, the littlefs uses a rather unique [COW](https://upload.wikimedia.org/wikipedia/commons/0/0c/Cow_female_black_white.jpg) +data structure that allows the filesystem to reuse unmodified parts of the +file without additional metadata pairs. + +First lets consider storing files in a simple linked-list. What happens when +append a block? We have to change the last block in the linked-list to point +to this new block, which means we have to copy out the last block, and change +the second-to-last block, and then the third-to-last, and so on until we've +copied out the entire file. + +``` +Exhibit A: A linked-list +.--------. .--------. .--------. .--------. .--------. .--------. +| data 0 |->| data 1 |->| data 2 |->| data 4 |->| data 5 |->| data 6 | +| | | | | | | | | | | | +| | | | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' '--------' +``` + +To get around this, the littlefs, at its heart, stores files backwards. Each +block points to its predecessor, with the first block containing no pointers. +If you think about this, it makes a bit of sense. Appending blocks just point +to their predecessor and no other blocks need to be updated. If we update +a block in the middle, we will need to copy out the blocks that follow, +but can reuse the blocks before the modified block. Since most file operations +either reset the file each write or append to files, this design avoids +copying the file in the most common cases. + +``` +Exhibit B: A backwards linked-list +.--------. .--------. .--------. .--------. .--------. .--------. +| data 0 |<-| data 1 |<-| data 2 |<-| data 4 |<-| data 5 |<-| data 6 | +| | | | | | | | | | | | +| | | | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' '--------' +``` + +However, a backwards linked-list does come with a rather glaring problem. +Iterating over a file _in order_ has a runtime of O(n^2). Gah! A quadratic +runtime to just _read_ a file? That's awful. Keep in mind reading files are +usually the most common filesystem operation. + +To avoid this problem, the littlefs uses a multilayered linked-list. For +every block that is divisible by a power of two, the block contains an +additional pointer that points back by that power of two. Another way of +thinking about this design is that there are actually many linked-lists +threaded together, with each linked-lists skipping an increasing number +of blocks. If you're familiar with data-structures, you may have also +recognized that this is a deterministic skip-list. + +To find the power of two factors efficiently, we can use the instruction +[count trailing zeros (CTZ)](https://en.wikipedia.org/wiki/Count_trailing_zeros), +which is where this linked-list's name comes from. + +``` +Exhibit C: A backwards CTZ linked-list +.--------. .--------. .--------. .--------. .--------. .--------. +| data 0 |<-| data 1 |<-| data 2 |<-| data 3 |<-| data 4 |<-| data 5 | +| |<-| |--| |<-| |--| | | | +| |<-| |--| |--| |--| | | | +'--------' '--------' '--------' '--------' '--------' '--------' +``` + +Taking exhibit C for example, here is the path from data block 5 to data +block 1. You can see how data block 3 was completely skipped: +``` +.--------. .--------. .--------. .--------. .--------. .--------. +| data 0 | | data 1 |<-| data 2 | | data 3 | | data 4 |<-| data 5 | +| | | | | |<-| |--| | | | +| | | | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' '--------' +``` + +The path to data block 0 is even more quick, requiring only two jumps: +``` +.--------. .--------. .--------. .--------. .--------. .--------. +| data 0 | | data 1 | | data 2 | | data 3 | | data 4 |<-| data 5 | +| | | | | | | | | | | | +| |<-| |--| |--| |--| | | | +'--------' '--------' '--------' '--------' '--------' '--------' +``` + +The CTZ linked-list has quite a few interesting properties. All of the pointers +in the block can be found by just knowing the index in the list of the current +block, and, with a bit of math, the amortized overhead for the linked-list is +only two pointers per block. Most importantly, the CTZ linked-list has a +worst case lookup runtime of O(logn), which brings the runtime of reading a +file down to O(n logn). Given that the constant runtime is divided by the +amount of data we can store in a block, this is pretty reasonable. + +Here is what it might look like to update a file stored with a CTZ linked-list: +``` + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 0 | + | file: 6 | file: 0 | + | size: 4 | size: 0 | + | xor: 3 | xor: 0 | + '---------'---------' + | + v + block 3 block 4 block 5 block 6 +.--------. .--------. .--------. .--------. +| data 0 |<-| data 1 |<-| data 2 |<-| data 3 | +| |<-| |--| | | | +| | | | | | | | +'--------' '--------' '--------' '--------' + +| update data in file +v + + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 0 | + | file: 6 | file: 0 | + | size: 4 | size: 0 | + | xor: 3 | xor: 0 | + '---------'---------' + | + v + block 3 block 4 block 5 block 6 +.--------. .--------. .--------. .--------. +| data 0 |<-| data 1 |<-| old |<-| old | +| |<-| |--| data 2 | | data 3 | +| | | | | | | | +'--------' '--------' '--------' '--------' + ^ ^ ^ + | | | block 7 block 8 block 9 block 10 + | | | .--------. .--------. .--------. .--------. + | | '----| new |<-| new |<-| new |<-| new | + | '----------------| data 2 |<-| data 3 |--| data 4 | | data 5 | + '------------------| |--| |--| | | | + '--------' '--------' '--------' '--------' + +| update metadata pair +v + + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 2 | + | file: 6 | file: 10| + | size: 4 | size: 6 | + | xor: 3 | xor: 14 | + '---------'---------' + | + | + block 3 block 4 block 5 block 6 | +.--------. .--------. .--------. .--------. | +| data 0 |<-| data 1 |<-| old |<-| old | | +| |<-| |--| data 2 | | data 3 | | +| | | | | | | | | +'--------' '--------' '--------' '--------' | + ^ ^ ^ v + | | | block 7 block 8 block 9 block 10 + | | | .--------. .--------. .--------. .--------. + | | '----| new |<-| new |<-| new |<-| new | + | '----------------| data 2 |<-| data 3 |--| data 4 | | data 5 | + '------------------| |--| |--| | | | + '--------' '--------' '--------' '--------' +``` + +## Block allocation + +So those two ideas provide the grounds for the filesystem. The metadata pairs +give us directories, and the CTZ linked-lists give us files. But this leaves +one big [elephant](https://upload.wikimedia.org/wikipedia/commons/3/37/African_Bush_Elephant.jpg) +of a question. How do we get those blocks in the first place? + +One common strategy is to store unallocated blocks in a big free list, and +initially the littlefs was designed with this in mind. By storing a reference +to the free list in every single metadata pair, additions to the free list +could be updated atomically at the same time the replacement blocks were +stored in the metadata pair. During boot, every metadata pair had to be +scanned to find the most recent free list, but once the list was found the +state of all free blocks becomes known. + +However, this approach had several issues: +- There was a lot of nuanced logic for adding blocks to the free list without + modifying the blocks, since the blocks remain active until the metadata is + updated. +- The free list had to support both additions and removals in fifo order while + minimizing block erases. +- The free list had to handle the case where the file system completely ran + out of blocks and may no longer be able to add blocks to the free list. +- If we used a revision count to track the most recently updated free list, + metadata blocks that were left unmodified were ticking time bombs that would + cause the system to go haywire if the revision count overflowed +- Every single metadata block wasted space to store these free list references. + +Actually, to simplify, this approach had one massive glaring issue: complexity. + +> Complexity leads to fallibility. +> Fallibility leads to unmaintainability. +> Unmaintainability leads to suffering. + +Or at least, complexity leads to increased code size, which is a problem +for embedded systems. + +In the end, the littlefs adopted more of a "drop it on the floor" strategy. +That is, the littlefs doesn't actually store information about which blocks +are free on the storage. The littlefs already stores which files _are_ in +use, so to find a free block, the littlefs just takes all of the blocks that +exist and subtract the blocks that are in use. + +Of course, it's not quite that simple. Most filesystems that adopt this "drop +it on the floor" strategy either rely on some properties inherent to the +filesystem, such as the cyclic-buffer structure of logging filesystems, +or use a bitmap or table stored in RAM to track free blocks, which scales +with the size of storage and is problematic when you have limited RAM. You +could iterate through every single block in storage and check it against +every single block in the filesystem on every single allocation, but that +would have an abhorrent runtime. + +So the littlefs compromises. It doesn't store a bitmap the size of the storage, +but it does store a little bit-vector that contains a fixed set lookahead +for block allocations. During a block allocation, the lookahead vector is +checked for any free blocks, if there are none, the lookahead region jumps +forward and the entire filesystem is scanned for free blocks. + +Here's what it might look like to allocate 4 blocks on a decently busy +filesystem with a 32bit lookahead and a total of +128 blocks (512Kbytes of storage if blocks are 4Kbyte): +``` +boot... lookahead: + fs blocks: fffff9fffffffffeffffffffffff0000 +scanning... lookahead: fffff9ff + fs blocks: fffff9fffffffffeffffffffffff0000 +alloc = 21 lookahead: fffffdff + fs blocks: fffffdfffffffffeffffffffffff0000 +alloc = 22 lookahead: ffffffff + fs blocks: fffffffffffffffeffffffffffff0000 +scanning... lookahead: fffffffe + fs blocks: fffffffffffffffeffffffffffff0000 +alloc = 63 lookahead: ffffffff + fs blocks: ffffffffffffffffffffffffffff0000 +scanning... lookahead: ffffffff + fs blocks: ffffffffffffffffffffffffffff0000 +scanning... lookahead: ffffffff + fs blocks: ffffffffffffffffffffffffffff0000 +scanning... lookahead: ffff0000 + fs blocks: ffffffffffffffffffffffffffff0000 +alloc = 112 lookahead: ffff8000 + fs blocks: ffffffffffffffffffffffffffff8000 +``` + +While this lookahead approach still has an asymptotic runtime of O(n^2) to +scan all of storage, the lookahead reduces the practical runtime to a +reasonable amount. Bit-vectors are surprisingly compact, given only 16 bytes, +the lookahead could track 128 blocks. For a 4Mbyte flash chip with 4Kbyte +blocks, the littlefs would only need 8 passes to scan the entire storage. + +The real benefit of this approach is just how much it simplified the design +of the littlefs. Deallocating blocks is as simple as simply forgetting they +exist, and there is absolutely no concern of bugs in the deallocation code +causing difficult to detect memory leaks. + +## Directories + +Now we just need directories to store our files. Since we already have +metadata blocks that store information about files, lets just use these +metadata blocks as the directories. Maybe turn the directories into linked +lists of metadata blocks so it isn't limited by the number of files that fit +in a single block. Add entries that represent other nested directories. +Drop "." and ".." entries, cause who needs them. Dust off our hands and +we now have a directory tree. + +``` + .--------. + |root dir| + | pair 0 | + | | + '--------' + .-' '-------------------------. + v v + .--------. .--------. .--------. + | dir A |------->| dir A | | dir B | + | pair 0 | | pair 1 | | pair 0 | + | | | | | | + '--------' '--------' '--------' + .-' '-. | .-' '-. + v v v v v +.--------. .--------. .--------. .--------. .--------. +| file C | | file D | | file E | | file F | | file G | +| | | | | | | | | | +| | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' +``` + +Unfortunately it turns out it's not that simple. See, iterating over a +directory tree isn't actually all that easy, especially when you're trying +to fit in a bounded amount of RAM, which rules out any recursive solution. +And since our block allocator involves iterating over the entire filesystem +tree, possibly multiple times in a single allocation, iteration needs to be +efficient. + +So, as a solution, the littlefs adopted a sort of threaded tree. Each +directory not only contains pointers to all of its children, but also a +pointer to the next directory. These pointers create a linked-list that +is threaded through all of the directories in the filesystem. Since we +only use this linked list to check for existance, the order doesn't actually +matter. As an added plus, we can repurpose the pointer for the individual +directory linked-lists and avoid using any additional space. + +``` + .--------. + |root dir|-. + | pair 0 | | + .--------| |-' + | '--------' + | .-' '-------------------------. + | v v + | .--------. .--------. .--------. + '->| dir A |------->| dir A |------->| dir B | + | pair 0 | | pair 1 | | pair 0 | + | | | | | | + '--------' '--------' '--------' + .-' '-. | .-' '-. + v v v v v +.--------. .--------. .--------. .--------. .--------. +| file C | | file D | | file E | | file F | | file G | +| | | | | | | | | | +| | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' +``` + +This threaded tree approach does come with a few tradeoffs. Now, anytime we +want to manipulate the directory tree, we find ourselves having to update two +pointers instead of one. For anyone familiar with creating atomic data +structures this should set off a whole bunch of red flags. + +But unlike the data structure guys, we can update a whole block atomically! So +as long as we're really careful (and cheat a little bit), we can still +manipulate the directory tree in a way that is resilient to power loss. + +Consider how we might add a new directory. Since both pointers that reference +it can come from the same directory, we only need a single atomic update to +finagle the directory into the filesystem: +``` + .--------. + |root dir|-. + | pair 0 | | +.--| |-' +| '--------' +| | +| v +| .--------. +'->| dir A | + | pair 0 | + | | + '--------' + +| create the new directory block +v + + .--------. + |root dir|-. + | pair 0 | | + .--| |-' + | '--------' + | | + | v + | .--------. +.--------. '->| dir A | +| dir B |---->| pair 0 | +| pair 0 | | | +| | '--------' +'--------' + +| update root to point to directory B +v + + .--------. + |root dir|-. + | pair 0 | | +.--------| |-' +| '--------' +| .-' '-. +| v v +| .--------. .--------. +'->| dir B |->| dir A | + | pair 0 | | pair 0 | + | | | | + '--------' '--------' +``` + +Note that even though directory B was added after directory A, we insert +directory B before directory A in the linked-list because it is convenient. + +Now how about removal: +``` + .--------. .--------. + |root dir|------->|root dir|-. + | pair 0 | | pair 1 | | +.--------| |--------| |-' +| '--------' '--------' +| .-' '-. | +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + | pair 0 | | pair 0 | | pair 0 | + | | | | | | + '--------' '--------' '--------' + +| update root to no longer contain directory B +v + + .--------. .--------. + |root dir|------------->|root dir|-. + | pair 0 | | pair 1 | | +.--| |--------------| |-' +| '--------' '--------' +| | | +| v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + | pair 0 | | pair 0 | | pair 0 | + | | | | | | + '--------' '--------' '--------' + +| remove directory B from the linked-list +v + + .--------. .--------. + |root dir|->|root dir|-. + | pair 0 | | pair 1 | | +.--| |--| |-' +| '--------' '--------' +| | | +| v v +| .--------. .--------. +'->| dir A |->| dir C | + | pair 0 | | pair 0 | + | | | | + '--------' '--------' +``` + +Wait, wait, wait, wait, wait, that's not atomic at all! If power is lost after +removing directory B from the root, directory B is still in the linked-list. +We've just created a memory leak! + +And to be honest, I don't have a clever solution for this case. As a +side-effect of using multiple pointers in the threaded tree, the littlefs +can end up with orphan blocks that have no parents and should have been +removed. + +To keep these orphan blocks from becoming a problem, the littlefs has a +deorphan step that simply iterates through every directory in the linked-list +and checks it against every directory entry in the filesystem to see if it +has a parent. The deorphan step occurs on the first block allocation after +boot, so orphans should never cause the littlefs to run out of storage +prematurely. + +And for my final trick, moving a directory: +``` + .--------. + |root dir|-. + | pair 0 | | +.--------| |-' +| '--------' +| .-' '-. +| v v +| .--------. .--------. +'->| dir A |->| dir B | + | pair 0 | | pair 0 | + | | | | + '--------' '--------' + +| update directory B to point to directory A +v + + .--------. + |root dir|-. + | pair 0 | | +.--------| |-' +| '--------' +| .-----' '-. +| | v +| | .--------. +| | .->| dir B | +| | | | pair 0 | +| | | | | +| | | '--------' +| | .-------' +| v v | +| .--------. | +'->| dir A |-' + | pair 0 | + | | + '--------' + +| update root to no longer contain directory A +v + .--------. + |root dir|-. + | pair 0 | | +.----| |-' +| '--------' +| | +| v +| .--------. +| .->| dir B | +| | | pair 0 | +| '--| |-. +| '--------' | +| | | +| v | +| .--------. | +'--->| dir A |-' + | pair 0 | + | | + '--------' +``` + +Note that once again we don't care about the ordering of directories in the +linked-list, so we can simply leave directories in their old positions. This +does make the diagrams a bit hard to draw, but the littlefs doesn't really +care. + +It's also worth noting that once again we have an operation that isn't actually +atomic. After we add directory A to directory B, we could lose power, leaving +directory A as a part of both the root directory and directory B. However, +there isn't anything inherent to the littlefs that prevents a directory from +having multiple parents, so in this case, we just allow that to happen. Extra +care is taken to only remove a directory from the linked-list if there are +no parents left in the filesystem. + +## Wear awareness + +So now that we have all of the pieces of a filesystem, we can look at a more +subtle attribute of embedded storage: The wear down of flash blocks. + +The first concern for the littlefs, is that prefectly valid blocks can suddenly +become unusable. As a nice side-effect of using a COW data-structure for files, +we can simply move on to a different block when a file write fails. All +modifications to files are performed in copies, so we will only replace the +old file when we are sure none of the new file has errors. Directories, on +the other hand, need a different strategy. + +The solution to directory corruption in the littlefs relies on the redundant +nature of the metadata pairs. If an error is detected during a write to one +of the metadata pairs, we seek out a new block to take its place. Once we find +a block without errors, we iterate through the directory tree, updating any +references to the corrupted metadata pair to point to the new metadata block. +Just like when we remove directories, we can lose power during this operation +and end up with a desynchronized metadata pair in our filesystem. And just like +when we remove directories, we leave the possibility of a desynchronized +metadata pair up to the deorphan step to clean up. + +Here's what encountering a directory error may look like with all of +the directories and directory pointers fully expanded: +``` + root dir + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 0 |--. + | | |-.| +.------| | |-|' +|.-----| | |-' +|| '---------'---------' +|| |||||'--------------------------------------------------. +|| ||||'-----------------------------------------. | +|| |||'-----------------------------. | | +|| ||'--------------------. | | | +|| |'-------. | | | | +|| v v v v v v +|| dir A dir B dir C +|| block 3 block 4 block 5 block 6 block 7 block 8 +|| .---------.---------. .---------.---------. .---------.---------. +|'->| rev: 1 | rev: 0 |->| rev: 1 | rev: 0 |->| rev: 1 | rev: 0 | +'-->| | |->| | |->| | | + | | | | | | | + | | | | | | | | | + '---------'---------' '---------'---------' '---------'---------' + +| update directory B +v + + root dir + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 0 |--. + | | |-.| +.------| | |-|' +|.-----| | |-' +|| '---------'---------' +|| |||||'--------------------------------------------------. +|| ||||'-----------------------------------------. | +|| |||'-----------------------------. | | +|| ||'--------------------. | | | +|| |'-------. | | | | +|| v v v v v v +|| dir A dir B dir C +|| block 3 block 4 block 5 block 6 block 7 block 8 +|| .---------.---------. .---------.---------. .---------.---------. +|'->| rev: 1 | rev: 0 |->| rev: 1 | rev: 2 |->| rev: 1 | rev: 0 | +'-->| | |->| | corrupt!|->| | | + | | | | | corrupt!| | | | + | | | | | corrupt!| | | | + '---------'---------' '---------'---------' '---------'---------' + +| oh no! corruption detected +v allocate a replacement block + + root dir + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 0 |--. + | | |-.| +.------| | |-|' +|.-----| | |-' +|| '---------'---------' +|| |||||'----------------------------------------------------. +|| ||||'-------------------------------------------. | +|| |||'-----------------------------. | | +|| ||'--------------------. | | | +|| |'-------. | | | | +|| v v v v v v +|| dir A dir B dir C +|| block 3 block 4 block 5 block 6 block 7 block 8 +|| .---------.---------. .---------.---------. .---------.---------. +|'->| rev: 1 | rev: 0 |->| rev: 1 | rev: 2 |--->| rev: 1 | rev: 0 | +'-->| | |->| | corrupt!|--->| | | + | | | | | corrupt!| .->| | | + | | | | | corrupt!| | | | | + '---------'---------' '---------'---------' | '---------'---------' + block 9 | + .---------. | + | rev: 2 |-' + | | + | | + | | + '---------' + +| update root directory to contain block 9 +v + + root dir + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 2 |--. + | | |-.| +.-----| | |-|' +|.----| | |-' +|| '---------'---------' +|| .--------'||||'----------------------------------------------. +|| | |||'-------------------------------------. | +|| | ||'-----------------------. | | +|| | |'------------. | | | +|| | | | | | | +|| v v v v v v +|| dir A dir B dir C +|| block 3 block 4 block 5 block 9 block 7 block 8 +|| .---------.---------. .---------. .---------. .---------.---------. +|'->| rev: 1 | rev: 0 |-->| rev: 1 |-| rev: 2 |--->| rev: 1 | rev: 0 | +'-->| | |-. | | | |--->| | | + | | | | | | | | .->| | | + | | | | | | | | | | | | + '---------'---------' | '---------' '---------' | '---------'---------' + | block 6 | + | .---------. | + '------------>| rev: 2 |-' + | corrupt!| + | corrupt!| + | corrupt!| + '---------' + +| remove corrupted block from linked-list +v + + root dir + block 1 block 2 + .---------.---------. + | rev: 1 | rev: 2 |--. + | | |-.| +.-----| | |-|' +|.----| | |-' +|| '---------'---------' +|| .--------'||||'-----------------------------------------. +|| | |||'--------------------------------. | +|| | ||'--------------------. | | +|| | |'-----------. | | | +|| | | | | | | +|| v v v v v v +|| dir A dir B dir C +|| block 3 block 4 block 5 block 9 block 7 block 8 +|| .---------.---------. .---------.---------. .---------.---------. +|'->| rev: 1 | rev: 2 |->| rev: 1 | rev: 2 |->| rev: 1 | rev: 0 | +'-->| | |->| | |->| | | + | | | | | | | | | + | | | | | | | | | + '---------'---------' '---------'---------' '---------'---------' +``` + +Also one question I've been getting is, what about the root directory? +It can't move so wouldn't the filesystem die as soon as the root blocks +develop errors? And you would be correct. So instead of storing the root +in the first few blocks of the storage, the root is actually pointed to +by the superblock. The superblock contains a few bits of static data, but +outside of when the filesystem is formatted, it is only updated when the root +develops errors and needs to be moved. + +## Wear leveling + +The second concern for the littlefs, is that blocks in the filesystem may wear +unevenly. In this situation, a filesystem may meet an early demise where +there are no more non-corrupted blocks that aren't in use. It may be entirely +possible that files were written once and left unmodified, wasting the +potential erase cycles of the blocks it sits on. + +Wear leveling is a term that describes distributing block writes evenly to +avoid the early termination of a flash part. There are typically two levels +of wear leveling: +1. Dynamic wear leveling - Blocks are distributed evenly during blocks writes. + Note that the issue with write-once files still exists in this case. +2. Static wear leveling - Unmodified blocks are evicted for new block writes. + This provides the longest lifetime for a flash device. + +Now, it's possible to use the revision count on metadata pairs to approximate +the wear of a metadata block. And combined with the COW nature of files, the +littlefs could provide a form of dynamic wear leveling. + +However, the littlefs does not. This is for a few reasons. Most notably, even +if the littlefs did implement dynamic wear leveling, this would still not +handle the case of write-once files, and near the end of the lifetime of a +flash device, you would likely end up with uneven wear on the blocks anyways. + +As a flash device reaches the end of its life, the metadata blocks will +naturally be the first to go since they are updated most often. In this +situation, the littlefs is designed to simply move on to another set of +metadata blocks. This travelling means that at the end of a flash device's +life, the filesystem will have worn the device down as evenly as a dynamic +wear leveling filesystem could anyways. Simply put, if the lifetime of flash +is a serious concern, static wear leveling is the only valid solution. + +This is a very important takeaway to note. If your storage stack uses highly +sensitive storage such as NAND flash. In most cases you are going to be better +off just using a [flash translation layer (FTL)](https://en.wikipedia.org/wiki/Flash_translation_layer). +NAND flash already has many limitations that make it poorly suited for an +embedded system: low erase cycles, very large blocks, errors that can develop +even during reads, errors that can develop during writes of neighboring blocks. +Managing sensitive storage such as NAND flash is out of scope for the littlefs. +The littlefs does have some properties that may be beneficial on top of a FTL, +such as limiting the number of writes where possible. But if you have the +storage requirements that necessitate the need of NAND flash, you should have +the RAM to match and just use an FTL or flash filesystem. + +## Summary + +So, to summarize: + +1. The littlefs is composed of directory blocks +2. Each directory is a linked-list of metadata pairs +3. These metadata pairs can be updated atomically by alternative which + metadata block is active +4. Directory blocks contain either references to other directories or files +5. Files are represented by copy-on-write CTZ linked-lists +6. The CTZ linked-lists support appending in O(1) and reading in O(n logn) +7. Blocks are allocated by scanning the filesystem for used blocks in a + fixed-size lookahead region is that stored in a bit-vector +8. To facilitate scanning the filesystem, all directories are part of a + linked-list that is threaded through the entire filesystem +9. If a block develops an error, the littlefs allocates a new block, and + moves the data and references of the old block to the new. +10. Any case where an atomic operation is not possible, it is taken care of + by a deorphan step that occurs on the first allocation after boot + +Welp, that's the little filesystem. Thanks for reading! + diff --git a/littlefs/LICENSE.md b/littlefs/LICENSE.md new file mode 100644 index 0000000000..59cd3f8a32 --- /dev/null +++ b/littlefs/LICENSE.md @@ -0,0 +1,165 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. diff --git a/littlefs/Makefile b/littlefs/Makefile new file mode 100644 index 0000000000..2c9e8bfafa --- /dev/null +++ b/littlefs/Makefile @@ -0,0 +1,57 @@ +TARGET = lfs + +CC = gcc +AR = ar +SIZE = size + +SRC += $(wildcard *.c emubd/*.c) +OBJ := $(SRC:.c=.o) +DEP := $(SRC:.c=.d) +ASM := $(SRC:.c=.s) + +TEST := $(patsubst tests/%.sh,%,$(wildcard tests/test_*)) + +ifdef DEBUG +CFLAGS += -O0 -g3 +else +CFLAGS += -Os +endif +ifdef WORD +CFLAGS += -m$(WORD) +endif +CFLAGS += -I. +CFLAGS += -std=c99 -Wall -pedantic + + +all: $(TARGET) + +asm: $(ASM) + +size: $(OBJ) + $(SIZE) -t $^ + +.SUFFIXES: +test: test_format test_dirs test_files test_seek test_parallel \ + test_alloc test_paths test_orphan test_corrupt +test_%: tests/test_%.sh + ./$< + +-include $(DEP) + +$(TARGET): $(OBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + +%.a: $(OBJ) + $(AR) rcs $@ $^ + +%.o: %.c + $(CC) -c -MMD $(CFLAGS) $< -o $@ + +%.s: %.c + $(CC) -S $(CFLAGS) $< -o $@ + +clean: + rm -f $(TARGET) + rm -f $(OBJ) + rm -f $(DEP) + rm -f $(ASM) diff --git a/littlefs/README.md b/littlefs/README.md new file mode 100644 index 0000000000..0ae1cc61ea --- /dev/null +++ b/littlefs/README.md @@ -0,0 +1,142 @@ +## The little filesystem + +A little fail-safe filesystem designed for embedded systems. + +``` + | | | .---._____ + .-----. | | +--|o |---| littlefs | +--| |---| | + '-----' '----------' + | | | +``` + +**Fail-safe** - The littlefs is designed to work consistently with random +power failures. During filesystem operations the storage on disk is always +kept in a valid state. The filesystem also has strong copy-on-write garuntees. +When updating a file, the original file will remain unmodified until the +file is closed, or sync is called. + +**Wear awareness** - While the littlefs does not implement static wear +leveling, the littlefs takes into account write errors reported by the +underlying block device and uses a limited form of dynamic wear leveling +to manage blocks that go bad during the lifetime of the filesystem. + +**Bounded ram/rom** - The littlefs is designed to work in a +limited amount of memory, recursion is avoided, and dynamic memory is kept +to a minimum. The littlefs allocates two fixed-size buffers for general +operations, and one fixed-size buffer per file. If there is only ever one file +in use, all memory can be provided statically and the littlefs can be used +in a system without dynamic memory. + +## Example + +Here's a simple example that updates a file named `boot_count` every time +main runs. The program can be interrupted at any time without losing track +of how many times it has been booted and without corrupting the filesystem: + +``` c +#include "lfs.h" + +// variables used by the filesystem +lfs_t lfs; +lfs_file_t file; + +// configuration of the filesystem is provided by this struct +const struct lfs_config cfg = { + // block device operations + .read = user_provided_block_device_read, + .prog = user_provided_block_device_prog, + .erase = user_provided_block_device_erase, + .sync = user_provided_block_device_sync, + + // block device configuration + .read_size = 16, + .prog_size = 16, + .block_size = 4096, + .block_count = 128, + .lookahead = 128, +}; + +// entry point +int main(void) { + // mount the filesystem + int err = lfs_mount(&lfs, &cfg); + + // reformat if we can't mount the filesystem + // this should only happen on the first boot + if (err) { + lfs_format(&lfs, &cfg); + lfs_mount(&lfs, &cfg); + } + + // read current count + uint32_t boot_count = 0; + lfs_file_open(&lfs, &file, "boot_count", LFS_O_RDWR | LFS_O_CREAT); + lfs_file_read(&lfs, &file, &boot_count, sizeof(boot_count)); + + // update boot count + boot_count += 1; + printf("boot_count: %ld\n", boot_count); + lfs_file_rewind(&lfs, &file); + lfs_file_write(&lfs, &file, &boot_count, sizeof(boot_count)); + + // remember the storage is not updated until the file is closed successfully + lfs_file_close(&lfs, &file); + + // release any resources we were using + lfs_unmount(&lfs); +} +``` + +## Usage + +Detailed documentation (or at least as much detail as is currently available) +can be cound in the comments in [lfs.h](lfs.h). + +As you may have noticed, the littlefs takes in a configuration structure that +defines how the filesystem operates. The configuration struct provides the +filesystem with the block device operations and dimensions, tweakable +parameters that tradeoff memory usage for performance, and optional +static buffers if the user wants to avoid dynamic memory. + +The state of the littlefs is stored in the `lfs_t` type which is left up +to the user to allocate, allowing multiple filesystems to be in use +simultaneously. With the `lfs_t` and configuration struct, a user can either +format a block device or mount the filesystem. + +Once mounted, the littlefs provides a full set of posix-like file and +directory functions, with the deviation that the allocation of filesystem +structures must be provided by the user. An important addition is that +no file updates will actually be written to disk until a sync or close +is called. + +## Other notes + +All littlefs have the potential to return a negative error code. The errors +can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h), +or an error returned by the user's block device operations. + +It should also be noted that the littlefs does not do anything to insure +that the data written to disk is machine portable. It should be fine as +long as the machines involved share endianness and don't have really +strange padding requirements. If the question does come up, the littlefs +metadata should be stored on disk in little-endian format. + +## Design + +the littlefs was developed with the goal of learning more about filesystem +design by tackling the relative unsolved problem of managing a robust +filesystem resilient to power loss on devices with limited RAM and ROM. +More detail on the solutions and tradeoffs incorporated into this filesystem +can be found in [DESIGN.md](DESIGN.md). + +## Testing + +The littlefs comes with a test suite designed to run on a pc using the +[emulated block device](emubd/lfs_emubd.h) found in the emubd directory. +The tests assume a linux environment and can be started with make: + +``` bash +make test +``` diff --git a/littlefs/emubd/lfs_emubd.c b/littlefs/emubd/lfs_emubd.c new file mode 100644 index 0000000000..b38d9d3361 --- /dev/null +++ b/littlefs/emubd/lfs_emubd.c @@ -0,0 +1,242 @@ +/* + * Block device emulated on standard files + * + * Copyright (c) 2017 Christopher Haster + * Distributed under the Apache 2.0 license + */ +#include "emubd/lfs_emubd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +// Block device emulated on existing filesystem +int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { + lfs_emubd_t *emu = cfg->context; + emu->cfg.read_size = cfg->read_size; + emu->cfg.prog_size = cfg->prog_size; + emu->cfg.block_size = cfg->block_size; + emu->cfg.block_count = cfg->block_count; + + // Allocate buffer for creating children files + size_t pathlen = strlen(path); + emu->path = malloc(pathlen + 1 + LFS_NAME_MAX + 1); + if (!emu->path) { + return -ENOMEM; + } + + strcpy(emu->path, path); + emu->path[pathlen] = '/'; + emu->child = &emu->path[pathlen+1]; + memset(emu->child, '\0', LFS_NAME_MAX+1); + + // Create directory if it doesn't exist + int err = mkdir(path, 0777); + if (err && errno != EEXIST) { + return -errno; + } + + // Load stats to continue incrementing + snprintf(emu->child, LFS_NAME_MAX, "stats"); + FILE *f = fopen(emu->path, "r"); + if (!f) { + return -errno; + } + + size_t res = fread(&emu->stats, sizeof(emu->stats), 1, f); + if (res < 1) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } + + return 0; +} + +void lfs_emubd_destroy(const struct lfs_config *cfg) { + lfs_emubd_sync(cfg); + + lfs_emubd_t *emu = cfg->context; + free(emu->path); +} + +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + lfs_emubd_t *emu = cfg->context; + uint8_t *data = buffer; + + // Check if read is valid + assert(off % cfg->read_size == 0); + assert(size % cfg->read_size == 0); + assert(block < cfg->block_count); + + // Zero out buffer for debugging + memset(data, 0, size); + + // Read data + snprintf(emu->child, LFS_NAME_MAX, "%x", block); + + FILE *f = fopen(emu->path, "rb"); + if (!f && errno != ENOENT) { + return -errno; + } + + if (f) { + int err = fseek(f, off, SEEK_SET); + if (err) { + return -errno; + } + + size_t res = fread(data, 1, size, f); + if (res < size && !feof(f)) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } + } + + emu->stats.read_count += 1; + return 0; +} + +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size) { + lfs_emubd_t *emu = cfg->context; + const uint8_t *data = buffer; + + // Check if write is valid + assert(off % cfg->prog_size == 0); + assert(size % cfg->prog_size == 0); + assert(block < cfg->block_count); + + // Program data + snprintf(emu->child, LFS_NAME_MAX, "%x", block); + + FILE *f = fopen(emu->path, "r+b"); + if (!f && errno != ENOENT) { + return -errno; + } + + // Check that file was erased + assert(f); + + int err = fseek(f, off, SEEK_SET); + if (err) { + return -errno; + } + + size_t res = fwrite(data, 1, size, f); + if (res < size) { + return -errno; + } + + err = fseek(f, off, SEEK_SET); + if (err) { + return -errno; + } + + uint8_t dat; + res = fread(&dat, 1, 1, f); + if (res < 1) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } + + emu->stats.prog_count += 1; + return 0; +} + +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { + lfs_emubd_t *emu = cfg->context; + + // Check if erase is valid + assert(block < cfg->block_count); + + // Erase the block + snprintf(emu->child, LFS_NAME_MAX, "%x", block); + struct stat st; + int err = stat(emu->path, &st); + if (err && errno != ENOENT) { + return -errno; + } + + if (!err && S_ISREG(st.st_mode)) { + int err = unlink(emu->path); + if (err) { + return -errno; + } + } + + if (err || S_ISREG(st.st_mode)) { + FILE *f = fopen(emu->path, "w"); + if (!f) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } + } + + emu->stats.erase_count += 1; + return 0; +} + +int lfs_emubd_sync(const struct lfs_config *cfg) { + lfs_emubd_t *emu = cfg->context; + + // Just write out info/stats for later lookup + snprintf(emu->child, LFS_NAME_MAX, "config"); + FILE *f = fopen(emu->path, "w"); + if (!f) { + return -errno; + } + + size_t res = fwrite(&emu->cfg, sizeof(emu->cfg), 1, f); + if (res < 1) { + return -errno; + } + + int err = fclose(f); + if (err) { + return -errno; + } + + snprintf(emu->child, LFS_NAME_MAX, "stats"); + f = fopen(emu->path, "w"); + if (!f) { + return -errno; + } + + res = fwrite(&emu->stats, sizeof(emu->stats), 1, f); + if (res < 1) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } + + return 0; +} + diff --git a/littlefs/emubd/lfs_emubd.h b/littlefs/emubd/lfs_emubd.h new file mode 100644 index 0000000000..083b2ce362 --- /dev/null +++ b/littlefs/emubd/lfs_emubd.h @@ -0,0 +1,78 @@ +/* + * Block device emulated on standard files + * + * Copyright (c) 2017 Christopher Haster + * Distributed under the Apache 2.0 license + */ +#ifndef LFS_EMUBD_H +#define LFS_EMUBD_H + +#include "lfs.h" +#include "lfs_util.h" + + +// Config options +#ifndef LFS_EMUBD_READ_SIZE +#define LFS_EMUBD_READ_SIZE 1 +#endif + +#ifndef LFS_EMUBD_PROG_SIZE +#define LFS_EMUBD_PROG_SIZE 1 +#endif + +#ifndef LFS_EMUBD_ERASE_SIZE +#define LFS_EMUBD_ERASE_SIZE 512 +#endif + +#ifndef LFS_EMUBD_TOTAL_SIZE +#define LFS_EMUBD_TOTAL_SIZE 524288 +#endif + + +// The emu bd state +typedef struct lfs_emubd { + char *path; + char *child; + + struct { + uint64_t read_count; + uint64_t prog_count; + uint64_t erase_count; + } stats; + + struct { + uint32_t read_size; + uint32_t prog_size; + uint32_t block_size; + uint32_t block_count; + } cfg; +} lfs_emubd_t; + + +// Create a block device using path for the directory to store blocks +int lfs_emubd_create(const struct lfs_config *cfg, const char *path); + +// Clean up memory associated with emu block device +void lfs_emubd_destroy(const struct lfs_config *cfg); + +// Read a block +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size); + +// Program a block +// +// The block must have previously been erased. +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size); + +// Erase a block +// +// A block must be erased before being programmed. The +// state of an erased block is undefined. +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block); + +// Sync the block device +int lfs_emubd_sync(const struct lfs_config *cfg); + + +#endif diff --git a/littlefs/lfs.c b/littlefs/lfs.c new file mode 100644 index 0000000000..a2d94eec61 --- /dev/null +++ b/littlefs/lfs.c @@ -0,0 +1,2146 @@ +/* + * The little filesystem + * + * Copyright (c) 2017 Christopher Haster + * Distributed under the Apache 2.0 license + */ +#include "lfs.h" +#include "lfs_util.h" + +#include +#include +#include + + +/// Caching block device operations /// +static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, + const lfs_cache_t *pcache, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + uint8_t *data = buffer; + assert(block < lfs->cfg->block_count); + + while (size > 0) { + if (pcache && block == pcache->block && off >= pcache->off && + off < pcache->off + lfs->cfg->prog_size) { + // is already in pcache? + lfs_size_t diff = lfs_min(size, + lfs->cfg->prog_size - (off-pcache->off)); + memcpy(data, &pcache->buffer[off-pcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + if (block == rcache->block && off >= rcache->off && + off < rcache->off + lfs->cfg->read_size) { + // is already in rcache? + lfs_size_t diff = lfs_min(size, + lfs->cfg->read_size - (off-rcache->off)); + memcpy(data, &rcache->buffer[off-rcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + if (off % lfs->cfg->read_size == 0 && size >= lfs->cfg->read_size) { + // bypass cache? + lfs_size_t diff = size - (size % lfs->cfg->read_size); + int err = lfs->cfg->read(lfs->cfg, block, off, data, diff); + if (err) { + return err; + } + + data += diff; + off += diff; + size -= diff; + continue; + } + + // load to cache, first condition can no longer fail + rcache->block = block; + rcache->off = off - (off % lfs->cfg->read_size); + int err = lfs->cfg->read(lfs->cfg, rcache->block, + rcache->off, rcache->buffer, lfs->cfg->read_size); + if (err) { + return err; + } + } + + return 0; +} + +static int lfs_cache_cmp(lfs_t *lfs, lfs_cache_t *rcache, + const lfs_cache_t *pcache, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size) { + const uint8_t *data = buffer; + + for (lfs_off_t i = 0; i < size; i++) { + uint8_t c; + int err = lfs_cache_read(lfs, rcache, pcache, + block, off+i, &c, 1); + if (err) { + return err; + } + + if (c != data[i]) { + return false; + } + } + + return true; +} + +static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache, + const lfs_cache_t *pcache, lfs_block_t block, + lfs_off_t off, lfs_size_t size, uint32_t *crc) { + for (lfs_off_t i = 0; i < size; i++) { + uint8_t c; + int err = lfs_cache_read(lfs, rcache, pcache, + block, off+i, &c, 1); + if (err) { + return err; + } + + lfs_crc(crc, &c, 1); + } + + return 0; +} + +static int lfs_cache_flush(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache) { + if (pcache->block != 0xffffffff) { + int err = lfs->cfg->prog(lfs->cfg, pcache->block, + pcache->off, pcache->buffer, lfs->cfg->prog_size); + if (err) { + return err; + } + + if (rcache) { + int res = lfs_cache_cmp(lfs, rcache, NULL, pcache->block, + pcache->off, pcache->buffer, lfs->cfg->prog_size); + if (res < 0) { + return res; + } + + if (!res) { + return LFS_ERR_CORRUPT; + } + } + + pcache->block = 0xffffffff; + } + + return 0; +} + +static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, + lfs_cache_t *rcache, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size) { + const uint8_t *data = buffer; + assert(block < lfs->cfg->block_count); + + while (size > 0) { + if (block == pcache->block && off >= pcache->off && + off < pcache->off + lfs->cfg->prog_size) { + // is already in pcache? + lfs_size_t diff = lfs_min(size, + lfs->cfg->prog_size - (off-pcache->off)); + memcpy(&pcache->buffer[off-pcache->off], data, diff); + + data += diff; + off += diff; + size -= diff; + + if (off % lfs->cfg->prog_size == 0) { + // eagerly flush out pcache if we fill up + int err = lfs_cache_flush(lfs, pcache, rcache); + if (err) { + return err; + } + } + + continue; + } + + // pcache must have been flushed, either by programming and + // entire block or manually flushing the pcache + assert(pcache->block == 0xffffffff); + + if (off % lfs->cfg->prog_size == 0 && + size >= lfs->cfg->prog_size) { + // bypass pcache? + lfs_size_t diff = size - (size % lfs->cfg->prog_size); + int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff); + if (err) { + return err; + } + + if (rcache) { + int res = lfs_cache_cmp(lfs, rcache, NULL, + block, off, data, diff); + if (res < 0) { + return res; + } + + if (!res) { + return LFS_ERR_CORRUPT; + } + } + + data += diff; + off += diff; + size -= diff; + continue; + } + + // prepare pcache, first condition can no longer fail + pcache->block = block; + pcache->off = off - (off % lfs->cfg->prog_size); + } + + return 0; +} + + +/// General lfs block device operations /// +static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + // if we ever do more than writes to alternating pairs, + // this may need to consider pcache + return lfs_cache_read(lfs, &lfs->rcache, NULL, + block, off, buffer, size); +} + +static int lfs_bd_prog(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size) { + return lfs_cache_prog(lfs, &lfs->pcache, NULL, + block, off, buffer, size); +} + +static int lfs_bd_cmp(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size) { + return lfs_cache_cmp(lfs, &lfs->rcache, NULL, block, off, buffer, size); +} + +static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, lfs_size_t size, uint32_t *crc) { + return lfs_cache_crc(lfs, &lfs->rcache, NULL, block, off, size, crc); +} + +static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { + return lfs->cfg->erase(lfs->cfg, block); +} + +static int lfs_bd_sync(lfs_t *lfs) { + lfs->rcache.block = 0xffffffff; + + int err = lfs_cache_flush(lfs, &lfs->pcache, NULL); + if (err) { + return err; + } + + return lfs->cfg->sync(lfs->cfg); +} + + +/// Internal operations predeclared here /// +int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir); +static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], + lfs_dir_t *parent, lfs_entry_t *entry); +static int lfs_relocate(lfs_t *lfs, + const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); +int lfs_deorphan(lfs_t *lfs); + + +/// Block allocator /// +static int lfs_alloc_lookahead(void *p, lfs_block_t block) { + lfs_t *lfs = p; + + lfs_block_t off = (block - lfs->free.start) % lfs->cfg->block_count; + if (off < lfs->cfg->lookahead) { + lfs->free.lookahead[off / 32] |= 1U << (off % 32); + } + + return 0; +} + +static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { + // deorphan if we haven't yet, only needed once after poweron + if (!lfs->deorphaned) { + int err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + + while (true) { + while (true) { + // check if we have looked at all blocks since last ack + if (lfs->free.start + lfs->free.off == lfs->free.end) { + LFS_WARN("No more free space %d", lfs->free.end); + return LFS_ERR_NOSPC; + } + + if (lfs->free.off >= lfs->cfg->lookahead) { + break; + } + + lfs_block_t off = lfs->free.off; + lfs->free.off += 1; + + if (!(lfs->free.lookahead[off / 32] & (1U << (off % 32)))) { + // found a free block + *block = (lfs->free.start + off) % lfs->cfg->block_count; + return 0; + } + } + + lfs->free.start += lfs->cfg->lookahead; + lfs->free.off = 0; + + // find mask of free blocks from tree + memset(lfs->free.lookahead, 0, lfs->cfg->lookahead/8); + int err = lfs_traverse(lfs, lfs_alloc_lookahead, lfs); + if (err) { + return err; + } + } +} + +static void lfs_alloc_ack(lfs_t *lfs) { + lfs->free.end = lfs->free.start + lfs->free.off + lfs->cfg->block_count; +} + + +/// Metadata pair and directory operations /// +static inline void lfs_pairswap(lfs_block_t pair[2]) { + lfs_block_t t = pair[0]; + pair[0] = pair[1]; + pair[1] = t; +} + +static inline bool lfs_pairisnull(const lfs_block_t pair[2]) { + return pair[0] == 0xffffffff || pair[1] == 0xffffffff; +} + +static inline int lfs_paircmp( + const lfs_block_t paira[2], + const lfs_block_t pairb[2]) { + return !(paira[0] == pairb[0] || paira[1] == pairb[1] || + paira[0] == pairb[1] || paira[1] == pairb[0]); +} + +static inline bool lfs_pairsync( + const lfs_block_t paira[2], + const lfs_block_t pairb[2]) { + return (paira[0] == pairb[0] && paira[1] == pairb[1]) || + (paira[0] == pairb[1] && paira[1] == pairb[0]); +} + +static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { + // allocate pair of dir blocks + for (int i = 0; i < 2; i++) { + int err = lfs_alloc(lfs, &dir->pair[i]); + if (err) { + return err; + } + } + + // rather than clobbering one of the blocks we just pretend + // the revision may be valid + int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4); + if (err) { + return err; + } + + // set defaults + dir->d.rev += 1; + dir->d.size = sizeof(dir->d)+4; + dir->d.tail[0] = -1; + dir->d.tail[1] = -1; + dir->off = sizeof(dir->d); + + // don't write out yet, let caller take care of that + return 0; +} + +static int lfs_dir_fetch(lfs_t *lfs, + lfs_dir_t *dir, const lfs_block_t pair[2]) { + // copy out pair, otherwise may be aliasing dir + const lfs_block_t tpair[2] = {pair[0], pair[1]}; + bool valid = false; + + // check both blocks for the most recent revision + for (int i = 0; i < 2; i++) { + struct lfs_disk_dir test; + int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); + if (err) { + return err; + } + + if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) { + continue; + } + + if ((0x7fffffff & test.size) < sizeof(test)+4 || + (0x7fffffff & test.size) > lfs->cfg->block_size) { + continue; + } + + uint32_t crc = 0xffffffff; + lfs_crc(&crc, &test, sizeof(test)); + err = lfs_bd_crc(lfs, tpair[i], sizeof(test), + (0x7fffffff & test.size) - sizeof(test), &crc); + if (err) { + return err; + } + + if (crc != 0) { + continue; + } + + valid = true; + + // setup dir in case it's valid + dir->pair[0] = tpair[(i+0) % 2]; + dir->pair[1] = tpair[(i+1) % 2]; + dir->off = sizeof(dir->d); + dir->d = test; + } + + if (!valid) { + LFS_ERROR("Corrupted dir pair at %d %d", tpair[0], tpair[1]); + return LFS_ERR_CORRUPT; + } + + return 0; +} + +struct lfs_region { + lfs_off_t oldoff; + lfs_size_t oldlen; + const void *newdata; + lfs_size_t newlen; +}; + +static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, + const struct lfs_region *regions, int count) { + dir->d.rev += 1; + lfs_pairswap(dir->pair); + for (int i = 0; i < count; i++) { + dir->d.size += regions[i].newlen - regions[i].oldlen; + } + + const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; + bool relocated = false; + + while (true) { + int err = lfs_bd_erase(lfs, dir->pair[0]); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + uint32_t crc = 0xffffffff; + lfs_crc(&crc, &dir->d, sizeof(dir->d)); + err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + int i = 0; + lfs_off_t oldoff = sizeof(dir->d); + lfs_off_t newoff = sizeof(dir->d); + while (newoff < (0x7fffffff & dir->d.size)-4) { + if (i < count && regions[i].oldoff == oldoff) { + lfs_crc(&crc, regions[i].newdata, regions[i].newlen); + int err = lfs_bd_prog(lfs, dir->pair[0], + newoff, regions[i].newdata, regions[i].newlen); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + oldoff += regions[i].oldlen; + newoff += regions[i].newlen; + i += 1; + } else { + uint8_t data; + int err = lfs_bd_read(lfs, oldpair[1], oldoff, &data, 1); + if (err) { + return err; + } + + lfs_crc(&crc, &data, 1); + err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + oldoff += 1; + newoff += 1; + } + } + + err = lfs_bd_prog(lfs, dir->pair[0], newoff, &crc, 4); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + err = lfs_bd_sync(lfs); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // successful commit, check checksum to make sure + crc = 0xffffffff; + err = lfs_bd_crc(lfs, dir->pair[0], 0, 0x7fffffff & dir->d.size, &crc); + if (err) { + return err; + } + + if (crc == 0) { + break; + } + +relocate: + //commit was corrupted + LFS_DEBUG("Bad block at %d", dir->pair[0]); + + // drop caches and prepare to relocate block + relocated = true; + lfs->pcache.block = 0xffffffff; + + // can't relocate superblock, filesystem is now frozen + if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { + LFS_WARN("Superblock %d has become unwritable", oldpair[0]); + return LFS_ERR_CORRUPT; + } + + // relocate half of pair + err = lfs_alloc(lfs, &dir->pair[0]); + if (err) { + return err; + } + } + + if (relocated) { + // update references if we relocated + LFS_DEBUG("Relocating %d %d to %d %d", + oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); + return lfs_relocate(lfs, oldpair, dir->pair); + } + + return 0; +} + +static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, + const lfs_entry_t *entry, const void *data) { + return lfs_dir_commit(lfs, dir, (struct lfs_region[]){ + {entry->off, sizeof(entry->d), &entry->d, sizeof(entry->d)}, + {entry->off+sizeof(entry->d), entry->d.len-sizeof(entry->d), + data, entry->d.len-sizeof(entry->d)} + }, data ? 2 : 1); +} + +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, const void *data) { + // check if we fit, if top bit is set we do not and move on + while (true) { + if (dir->d.size + entry->d.len <= lfs->cfg->block_size) { + entry->off = dir->d.size - 4; + return lfs_dir_commit(lfs, dir, (struct lfs_region[]){ + {entry->off, 0, &entry->d, sizeof(entry->d)}, + {entry->off, 0, data, entry->d.len - sizeof(entry->d)} + }, 2); + } + + // we need to allocate a new dir block + if (!(0x80000000 & dir->d.size)) { + lfs_dir_t newdir; + int err = lfs_dir_alloc(lfs, &newdir); + if (err) { + return err; + } + + newdir.d.tail[0] = dir->d.tail[0]; + newdir.d.tail[1] = dir->d.tail[1]; + entry->off = newdir.d.size - 4; + err = lfs_dir_commit(lfs, &newdir, (struct lfs_region[]){ + {entry->off, 0, &entry->d, sizeof(entry->d)}, + {entry->off, 0, data, entry->d.len - sizeof(entry->d)} + }, 2); + if (err) { + return err; + } + + dir->d.size |= 0x80000000; + dir->d.tail[0] = newdir.pair[0]; + dir->d.tail[1] = newdir.pair[1]; + return lfs_dir_commit(lfs, dir, NULL, 0); + } + + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + } +} + +static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { + // either shift out the one entry or remove the whole dir block + if (dir->d.size == sizeof(dir->d)+4) { + lfs_dir_t pdir; + int res = lfs_pred(lfs, dir->pair, &pdir); + if (res < 0) { + return res; + } + + if (!(pdir.d.size & 0x80000000)) { + return lfs_dir_commit(lfs, dir, (struct lfs_region[]){ + {entry->off, entry->d.len, NULL, 0}, + }, 1); + } else { + pdir.d.tail[0] = dir->d.tail[0]; + pdir.d.tail[1] = dir->d.tail[1]; + return lfs_dir_commit(lfs, dir, NULL, 0); + } + } else { + return lfs_dir_commit(lfs, dir, (struct lfs_region[]){ + {entry->off, entry->d.len, NULL, 0}, + }, 1); + } +} + +static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { + while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { + if (!(0x80000000 & dir->d.size)) { + entry->off = dir->off; + return LFS_ERR_NOENT; + } + + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + + dir->off = sizeof(dir->d); + dir->pos += sizeof(dir->d) + 4; + } + + int err = lfs_bd_read(lfs, dir->pair[0], dir->off, + &entry->d, sizeof(entry->d)); + if (err) { + return err; + } + + dir->off += entry->d.len; + dir->pos += entry->d.len; + entry->off = dir->off - entry->d.len; + return 0; +} + +static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, const char **path) { + const char *pathname = *path; + size_t pathlen; + + while (true) { + nextname: + // skip slashes + pathname += strspn(pathname, "/"); + pathlen = strcspn(pathname, "/"); + + // skip '.' and root '..' + if ((pathlen == 1 && memcmp(pathname, ".", 1) == 0) || + (pathlen == 2 && memcmp(pathname, "..", 2) == 0)) { + pathname += pathlen; + goto nextname; + } + + // skip if matched by '..' in name + const char *suffix = pathname + pathlen; + size_t sufflen; + int depth = 1; + while (true) { + suffix += strspn(suffix, "/"); + sufflen = strcspn(suffix, "/"); + if (sufflen == 0) { + break; + } + + if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { + depth -= 1; + if (depth == 0) { + pathname = suffix + sufflen; + goto nextname; + } + } else { + depth += 1; + } + + suffix += sufflen; + } + + // find path + while (true) { + int err = lfs_dir_next(lfs, dir, entry); + if (err) { + return err; + } + + if ((entry->d.type != LFS_TYPE_REG && + entry->d.type != LFS_TYPE_DIR) || + entry->d.name != pathlen) { + continue; + } + + int res = lfs_bd_cmp(lfs, dir->pair[0], + entry->off + sizeof(entry->d), pathname, pathlen); + if (res < 0) { + return res; + } + + // found match + if (res) { + break; + } + } + + pathname += pathlen; + pathname += strspn(pathname, "/"); + if (pathname[0] == '\0') { + return 0; + } + + // continue on if we hit a directory + if (entry->d.type != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + int err = lfs_dir_fetch(lfs, dir, entry->d.u.dir); + if (err) { + return err; + } + + *path = pathname; + } + + return 0; +} + + +/// Top level directory operations /// +int lfs_mkdir(lfs_t *lfs, const char *path) { + // fetch parent directory + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err != LFS_ERR_NOENT) { + return err ? err : LFS_ERR_EXISTS; + } + + // build up new directory + lfs_alloc_ack(lfs); + + lfs_dir_t dir; + err = lfs_dir_alloc(lfs, &dir); + if (err) { + return err; + } + dir.d.tail[0] = cwd.d.tail[0]; + dir.d.tail[1] = cwd.d.tail[1]; + + err = lfs_dir_commit(lfs, &dir, NULL, 0); + if (err) { + return err; + } + + entry.d.type = LFS_TYPE_DIR; + entry.d.name = strlen(path); + entry.d.len = sizeof(entry.d) + entry.d.name; + entry.d.u.dir[0] = dir.pair[0]; + entry.d.u.dir[1] = dir.pair[1]; + + cwd.d.tail[0] = dir.pair[0]; + cwd.d.tail[1] = dir.pair[1]; + + err = lfs_dir_append(lfs, &cwd, &entry, path); + if (err) { + return err; + } + + lfs_alloc_ack(lfs); + return 0; +} + +int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { + dir->pair[0] = lfs->root[0]; + dir->pair[1] = lfs->root[1]; + + int err = lfs_dir_fetch(lfs, dir, dir->pair); + if (err) { + return err; + } + + if (strspn(path, "/.") == strlen(path)) { + // can only be something like '/././../.' + dir->head[0] = dir->pair[0]; + dir->head[1] = dir->pair[1]; + dir->pos = sizeof(dir->d) - 2; + dir->off = sizeof(dir->d); + return 0; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, dir, &entry, &path); + if (err) { + return err; + } else if (entry.d.type != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + err = lfs_dir_fetch(lfs, dir, entry.d.u.dir); + if (err) { + return err; + } + + // setup head dir + // special offset for '.' and '..' + dir->head[0] = dir->pair[0]; + dir->head[1] = dir->pair[1]; + dir->pos = sizeof(dir->d) - 2; + dir->off = sizeof(dir->d); + return 0; +} + +int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { + // do nothing, dir is always synchronized + return 0; +} + +int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { + memset(info, 0, sizeof(*info)); + + // special offset for '.' and '..' + if (dir->pos == sizeof(dir->d) - 2) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, "."); + dir->pos += 1; + return 1; + } else if (dir->pos == sizeof(dir->d) - 1) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, ".."); + dir->pos += 1; + return 1; + } + + lfs_entry_t entry; + while (true) { + int err = lfs_dir_next(lfs, dir, &entry); + if (err) { + return (err == LFS_ERR_NOENT) ? 0 : err; + } + + if (entry.d.type == LFS_TYPE_REG || + entry.d.type == LFS_TYPE_DIR) { + break; + } + } + + info->type = entry.d.type; + if (info->type == LFS_TYPE_REG) { + info->size = entry.d.u.file.size; + } + + int err = lfs_bd_read(lfs, dir->pair[0], entry.off + sizeof(entry.d), + info->name, entry.d.name); + if (err) { + return err; + } + + return 1; +} + +int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { + // simply walk from head dir + int err = lfs_dir_rewind(lfs, dir); + if (err) { + return err; + } + dir->pos = off; + + while (off > (0x7fffffff & dir->d.size)) { + off -= 0x7fffffff & dir->d.size; + if (!(0x80000000 & dir->d.size)) { + return LFS_ERR_INVAL; + } + + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + } + + dir->off = off; + return 0; +} + +lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { + return dir->pos; +} + +int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { + // reload the head dir + int err = lfs_dir_fetch(lfs, dir, dir->head); + if (err) { + return err; + } + + dir->pair[0] = dir->head[0]; + dir->pair[1] = dir->head[1]; + dir->pos = sizeof(dir->d) - 2; + dir->off = sizeof(dir->d); + return 0; +} + + +/// File index list operations /// +static int lfs_index(lfs_t *lfs, lfs_off_t *off) { + lfs_off_t i = 0; + lfs_size_t words = lfs->cfg->block_size / 4; + + while (*off >= lfs->cfg->block_size) { + i += 1; + *off -= lfs->cfg->block_size; + *off += 4*lfs_min(lfs_ctz(i)+1, words-1); + } + + return i; +} + +static int lfs_index_find(lfs_t *lfs, + lfs_cache_t *rcache, const lfs_cache_t *pcache, + lfs_block_t head, lfs_size_t size, + lfs_size_t pos, lfs_block_t *block, lfs_off_t *off) { + if (size == 0) { + *block = -1; + *off = 0; + return 0; + } + + lfs_off_t current = lfs_index(lfs, &(lfs_off_t){size-1}); + lfs_off_t target = lfs_index(lfs, &pos); + lfs_size_t words = lfs->cfg->block_size / 4; + + while (current > target) { + lfs_size_t skip = lfs_min( + lfs_npw2(current-target+1) - 1, + lfs_min(lfs_ctz(current)+1, words-1) - 1); + + int err = lfs_cache_read(lfs, rcache, pcache, head, 4*skip, &head, 4); + if (err) { + return err; + } + + current -= 1 << skip; + } + + *block = head; + *off = pos; + return 0; +} + +static int lfs_index_extend(lfs_t *lfs, + lfs_cache_t *rcache, lfs_cache_t *pcache, + lfs_block_t head, lfs_size_t size, + lfs_off_t *block, lfs_block_t *off) { + while (true) { + // go ahead and grab a block + int err = lfs_alloc(lfs, block); + if (err) { + return err; + } + + err = lfs_bd_erase(lfs, *block); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + if (size == 0) { + *off = 0; + return 0; + } + + size -= 1; + lfs_off_t index = lfs_index(lfs, &size); + size += 1; + + // just copy out the last block if it is incomplete + if (size != lfs->cfg->block_size) { + for (lfs_off_t i = 0; i < size; i++) { + uint8_t data; + int err = lfs_cache_read(lfs, rcache, NULL, head, i, &data, 1); + if (err) { + return err; + } + + err = lfs_cache_prog(lfs, pcache, rcache, *block, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + *off = size; + return 0; + } + + // append block + index += 1; + lfs_size_t words = lfs->cfg->block_size / 4; + lfs_size_t skips = lfs_min(lfs_ctz(index)+1, words-1); + + for (lfs_off_t i = 0; i < skips; i++) { + int err = lfs_cache_prog(lfs, pcache, rcache, + *block, 4*i, &head, 4); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + if (i != skips-1) { + err = lfs_cache_read(lfs, rcache, NULL, head, 4*i, &head, 4); + if (err) { + return err; + } + } + } + + *off = 4*skips; + return 0; + +relocate: + LFS_DEBUG("Bad block at %d", *block); + + // just clear cache and try a new block + pcache->block = 0xffffffff; + } +} + +static int lfs_index_traverse(lfs_t *lfs, + lfs_cache_t *rcache, const lfs_cache_t *pcache, + lfs_block_t head, lfs_size_t size, + int (*cb)(void*, lfs_block_t), void *data) { + if (size == 0) { + return 0; + } + + lfs_off_t index = lfs_index(lfs, &(lfs_off_t){size-1}); + + while (true) { + int err = cb(data, head); + if (err) { + return err; + } + + if (index == 0) { + return 0; + } + + err = lfs_cache_read(lfs, rcache, pcache, head, 0, &head, 4); + if (err) { + return err; + } + + index -= 1; + } + + return 0; +} + + +/// Top level file operations /// +int lfs_file_open(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags) { + // allocate entry for file if it doesn't exist + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + if (err == LFS_ERR_NOENT) { + if (!(flags & LFS_O_CREAT)) { + return LFS_ERR_NOENT; + } + + // create entry to remember name + entry.d.type = LFS_TYPE_REG; + entry.d.name = strlen(path); + entry.d.len = sizeof(entry.d) + entry.d.name; + entry.d.u.file.head = -1; + entry.d.u.file.size = 0; + err = lfs_dir_append(lfs, &cwd, &entry, path); + if (err) { + return err; + } + } else if (entry.d.type == LFS_TYPE_DIR) { + return LFS_ERR_ISDIR; + } else if (flags & LFS_O_EXCL) { + return LFS_ERR_EXISTS; + } + + // setup file struct + file->pair[0] = cwd.pair[0]; + file->pair[1] = cwd.pair[1]; + file->poff = entry.off; + file->head = entry.d.u.file.head; + file->size = entry.d.u.file.size; + file->flags = flags; + file->pos = 0; + + if (flags & LFS_O_TRUNC) { + file->head = -1; + file->size = 0; + } + + // allocate buffer if needed + file->cache.block = 0xffffffff; + if (lfs->cfg->file_buffer) { + file->cache.buffer = lfs->cfg->file_buffer; + } else if ((file->flags & 3) == LFS_O_RDONLY) { + file->cache.buffer = malloc(lfs->cfg->read_size); + if (!file->cache.buffer) { + return LFS_ERR_NOMEM; + } + } else { + file->cache.buffer = malloc(lfs->cfg->prog_size); + if (!file->cache.buffer) { + return LFS_ERR_NOMEM; + } + } + + // add to list of files + file->next = lfs->files; + lfs->files = file; + + return 0; +} + +int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { + int err = lfs_file_sync(lfs, file); + + // remove from list of files + for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; + } + } + + // clean up memory + if (!lfs->cfg->file_buffer) { + free(file->cache.buffer); + } + + return err; +} + +static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { +relocate: + LFS_DEBUG("Bad block at %d", file->block); + + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); + if (err) { + return err; + } + + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); + if (err) { + return err; + } + + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; + + file->block = nblock; + return 0; +} + +static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_READING) { + // just drop read cache + file->cache.block = 0xffffffff; + file->flags &= ~LFS_F_READING; + } + + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; + + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; + lfs->rcache.block = 0xffffffff; + + while (file->pos < file->size) { + // copy over a byte at a time, leave it up to caching + // to make this efficient + uint8_t data; + lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); + if (res < 0) { + return res; + } + + res = lfs_file_write(lfs, file, &data, 1); + if (res < 0) { + return res; + } + + // keep our reference to the rcache in sync + if (lfs->rcache.block != 0xffffffff) { + orig.cache.block = 0xffffffff; + lfs->rcache.block = 0xffffffff; + } + } + + // write out what we have + while (true) { + int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + break; +relocate: + err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } + } + + // actual file updates + file->head = file->block; + file->size = file->pos; + file->flags &= ~LFS_F_WRITING; + file->flags |= LFS_F_DIRTY; + + file->pos = pos; + } + + return 0; +} + +int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + + if ((file->flags & LFS_F_DIRTY) && !lfs_pairisnull(file->pair)) { + // update dir entry + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, file->pair); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = file->poff}; + err = lfs_bd_read(lfs, cwd.pair[0], entry.off, + &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + + if (entry.d.type != LFS_TYPE_REG) { + // sanity check valid entry + return LFS_ERR_INVAL; + } + + entry.d.u.file.head = file->head; + entry.d.u.file.size = file->size; + + err = lfs_dir_update(lfs, &cwd, &entry, NULL); + if (err) { + return err; + } + + file->flags &= ~LFS_F_DIRTY; + } + + return 0; +} + +lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size) { + uint8_t *data = buffer; + lfs_size_t nsize = size; + + if ((file->flags & 3) == LFS_O_WRONLY) { + return LFS_ERR_INVAL; + } + + if (file->flags & LFS_F_WRITING) { + // flush out any writes + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + } + + size = lfs_min(size, file->size - file->pos); + nsize = size; + + while (nsize > 0) { + // check if we need a new block + if (!(file->flags & LFS_F_READING) || + file->off == lfs->cfg->block_size) { + int err = lfs_index_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos, &file->block, &file->off); + if (err) { + return err; + } + + file->flags |= LFS_F_READING; + } + + // read as much as we can in current block + lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); + int err = lfs_cache_read(lfs, &file->cache, NULL, + file->block, file->off, data, diff); + if (err) { + return err; + } + + file->pos += diff; + file->off += diff; + data += diff; + nsize -= diff; + } + + return size; +} + +lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size) { + const uint8_t *data = buffer; + lfs_size_t nsize = size; + + if ((file->flags & 3) == LFS_O_RDONLY) { + return LFS_ERR_INVAL; + } + + if (file->flags & LFS_F_READING) { + // drop any reads + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + } + + if ((file->flags & LFS_O_APPEND) && file->pos < file->size) { + file->pos = file->size; + } + + while (nsize > 0) { + // check if we need a new block + if (!(file->flags & LFS_F_WRITING) || + file->off == lfs->cfg->block_size) { + if (!(file->flags & LFS_F_WRITING)) { + // find out which block we're extending from + int err = lfs_index_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos, &file->block, &file->off); + if (err) { + return err; + } + + // mark cache as dirty since we may have read data into it + file->cache.block = 0xffffffff; + file->flags |= LFS_F_WRITING; + } + + // extend file with new blocks + lfs_alloc_ack(lfs); + int err = lfs_index_extend(lfs, &lfs->rcache, &file->cache, + file->block, file->pos, + &file->block, &file->off); + if (err) { + return err; + } + } + + // program as much as we can in current block + lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); + while (true) { + int err = lfs_cache_prog(lfs, &file->cache, &lfs->rcache, + file->block, file->off, data, diff); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + break; +relocate: + err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } + } + + file->pos += diff; + file->off += diff; + data += diff; + nsize -= diff; + + lfs_alloc_ack(lfs); + } + + return size; +} + +lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, + lfs_soff_t off, int whence) { + // write out everything beforehand, may be noop if rdonly + int err = lfs_file_flush(lfs, file); + if (err) { + return err; + } + + // update pos + lfs_off_t pos = file->pos; + + if (whence == LFS_SEEK_SET) { + file->pos = off; + } else if (whence == LFS_SEEK_CUR) { + file->pos = file->pos + off; + } else if (whence == LFS_SEEK_END) { + file->pos = file->size + off; + } + + return pos; +} + +lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file) { + return file->pos; +} + +int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file) { + lfs_soff_t res = lfs_file_seek(lfs, file, 0, LFS_SEEK_SET); + if (res < 0) { + return res; + } + + return 0; +} + +lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { + return lfs_max(file->pos, file->size); +} + + +/// General fs oprations /// +int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + memset(info, 0, sizeof(*info)); + info->type = entry.d.type; + if (info->type == LFS_TYPE_REG) { + info->size = entry.d.u.file.size; + } + + err = lfs_bd_read(lfs, cwd.pair[0], entry.off + sizeof(entry.d), + info->name, entry.d.name); + if (err) { + return err; + } + + return 0; +} + +int lfs_remove(lfs_t *lfs, const char *path) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + lfs_dir_t dir; + if (entry.d.type == LFS_TYPE_DIR) { + // must be empty before removal, checking size + // without masking top bit checks for any case where + // dir is not empty + int err = lfs_dir_fetch(lfs, &dir, entry.d.u.dir); + if (err) { + return err; + } else if (dir.d.size != sizeof(dir.d)+4) { + return LFS_ERR_INVAL; + } + } + + // remove the entry + err = lfs_dir_remove(lfs, &cwd, &entry); + if (err) { + return err; + } + + // shift over any files that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, cwd.pair) == 0) { + if (f->poff == entry.off) { + f->pair[0] = 0xffffffff; + f->pair[1] = 0xffffffff; + } else if (f->poff > entry.off) { + f->poff -= entry.d.len; + } + } + } + + // if we were a directory, just run a deorphan step, this should + // collect us, although is expensive + if (entry.d.type == LFS_TYPE_DIR) { + int err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + + return 0; +} + +int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { + // find old entry + lfs_dir_t oldcwd; + int err = lfs_dir_fetch(lfs, &oldcwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t oldentry; + err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); + if (err) { + return err; + } + + // allocate new entry + lfs_dir_t newcwd; + err = lfs_dir_fetch(lfs, &newcwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t preventry; + err = lfs_dir_find(lfs, &newcwd, &preventry, &newpath); + if (err && err != LFS_ERR_NOENT) { + return err; + } + bool prevexists = (err != LFS_ERR_NOENT); + + // must have same type + if (prevexists && preventry.d.type != oldentry.d.type) { + return LFS_ERR_INVAL; + } + + lfs_dir_t dir; + if (prevexists && preventry.d.type == LFS_TYPE_DIR) { + // must be empty before removal, checking size + // without masking top bit checks for any case where + // dir is not empty + int err = lfs_dir_fetch(lfs, &dir, preventry.d.u.dir); + if (err) { + return err; + } else if (dir.d.size != sizeof(dir.d)+4) { + return LFS_ERR_INVAL; + } + } + + // move to new location + lfs_entry_t newentry = preventry; + newentry.d = oldentry.d; + newentry.d.name = strlen(newpath); + newentry.d.len = sizeof(newentry.d) + newentry.d.name; + + if (prevexists) { + int err = lfs_dir_update(lfs, &newcwd, &newentry, newpath); + if (err) { + return err; + } + } else { + int err = lfs_dir_append(lfs, &newcwd, &newentry, newpath); + if (err) { + return err; + } + } + + // fetch again in case newcwd == oldcwd + err = lfs_dir_fetch(lfs, &oldcwd, oldcwd.pair); + if (err) { + return err; + } + + err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); + if (err) { + return err; + } + + // remove from old location + err = lfs_dir_remove(lfs, &oldcwd, &oldentry); + if (err) { + return err; + } + + // shift over any files that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, oldcwd.pair) == 0) { + if (f->poff == oldentry.off) { + f->pair[0] = 0xffffffff; + f->pair[1] = 0xffffffff; + } else if (f->poff > oldentry.off) { + f->poff -= oldentry.d.len; + } + } + } + + // if we were a directory, just run a deorphan step, this should + // collect us, although is expensive + if (prevexists && preventry.d.type == LFS_TYPE_DIR) { + int err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + + return 0; +} + + +/// Filesystem operations /// +static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { + lfs->cfg = cfg; + + // setup read cache + lfs->rcache.block = 0xffffffff; + if (lfs->cfg->read_buffer) { + lfs->rcache.buffer = lfs->cfg->read_buffer; + } else { + lfs->rcache.buffer = malloc(lfs->cfg->read_size); + if (!lfs->rcache.buffer) { + return LFS_ERR_NOMEM; + } + } + + // setup program cache + lfs->pcache.block = 0xffffffff; + if (lfs->cfg->prog_buffer) { + lfs->pcache.buffer = lfs->cfg->prog_buffer; + } else { + lfs->pcache.buffer = malloc(lfs->cfg->prog_size); + if (!lfs->pcache.buffer) { + return LFS_ERR_NOMEM; + } + } + + // setup lookahead + if (lfs->cfg->lookahead_buffer) { + lfs->free.lookahead = lfs->cfg->lookahead_buffer; + } else { + lfs->free.lookahead = malloc(lfs->cfg->lookahead/8); + if (!lfs->free.lookahead) { + return LFS_ERR_NOMEM; + } + } + + // setup default state + lfs->root[0] = 0xffffffff; + lfs->root[1] = 0xffffffff; + lfs->files = NULL; + lfs->deorphaned = false; + + return 0; +} + +static int lfs_deinit(lfs_t *lfs) { + // free allocated memory + if (!lfs->cfg->read_buffer) { + free(lfs->rcache.buffer); + } + + if (!lfs->cfg->prog_buffer) { + free(lfs->pcache.buffer); + } + + if (!lfs->cfg->lookahead_buffer) { + free(lfs->free.lookahead); + } + + return 0; +} + +int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { + int err = lfs_init(lfs, cfg); + if (err) { + return err; + } + + // create free lookahead + memset(lfs->free.lookahead, 0, lfs->cfg->lookahead/8); + lfs->free.start = 0; + lfs->free.off = 0; + lfs->free.end = lfs->free.start + lfs->cfg->block_count; + + // create superblock dir + lfs_alloc_ack(lfs); + lfs_dir_t superdir; + err = lfs_dir_alloc(lfs, &superdir); + if (err) { + return err; + } + + // write root directory + lfs_dir_t root; + err = lfs_dir_alloc(lfs, &root); + if (err) { + return err; + } + + err = lfs_dir_commit(lfs, &root, NULL, 0); + if (err) { + return err; + } + + lfs->root[0] = root.pair[0]; + lfs->root[1] = root.pair[1]; + + // write superblocks + lfs_superblock_t superblock = { + .off = sizeof(superdir.d), + .d.type = LFS_TYPE_SUPERBLOCK, + .d.name = sizeof(superblock.d.magic), + .d.len = sizeof(superblock.d), + .d.version = 0x00010001, + .d.magic = {"littlefs"}, + .d.block_size = lfs->cfg->block_size, + .d.block_count = lfs->cfg->block_count, + .d.root = {lfs->root[0], lfs->root[1]}, + }; + superdir.d.tail[0] = root.pair[0]; + superdir.d.tail[1] = root.pair[1]; + superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4; + + // write both pairs to be safe + bool valid = false; + for (int i = 0; i < 2; i++) { + int err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ + {sizeof(superdir.d), sizeof(superblock.d), + &superblock.d, sizeof(superblock.d)} + }, 1); + if (err && err != LFS_ERR_CORRUPT) { + return err; + } + + valid = valid || !err; + } + + if (!valid) { + return LFS_ERR_CORRUPT; + } + + // sanity check that fetch works + err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + lfs_alloc_ack(lfs); + return lfs_deinit(lfs); +} + +int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { + int err = lfs_init(lfs, cfg); + if (err) { + return err; + } + + // setup free lookahead + lfs->free.start = -lfs->cfg->lookahead; + lfs->free.off = lfs->cfg->lookahead; + lfs->free.end = lfs->free.start + lfs->cfg->block_count; + + // load superblock + lfs_dir_t dir; + lfs_superblock_t superblock; + err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (!err) { + err = lfs_bd_read(lfs, dir.pair[0], + sizeof(dir.d), &superblock.d, sizeof(superblock.d)); + + lfs->root[0] = superblock.d.root[0]; + lfs->root[1] = superblock.d.root[1]; + } + + if (err == LFS_ERR_CORRUPT || + memcmp(superblock.d.magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", dir.pair[0], dir.pair[1]); + return LFS_ERR_CORRUPT; + } + + if (superblock.d.version > (0x00010001 | 0x0000ffff)) { + LFS_ERROR("Invalid version %d.%d\n", + 0xffff & (superblock.d.version >> 16), + 0xffff & (superblock.d.version >> 0)); + return LFS_ERR_INVAL; + } + + return err; +} + +int lfs_unmount(lfs_t *lfs) { + return lfs_deinit(lfs); +} + + +/// Littlefs specific operations /// +int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { + if (lfs_pairisnull(lfs->root)) { + return 0; + } + + // iterate over metadata pairs + lfs_dir_t dir; + lfs_entry_t entry; + lfs_block_t cwd[2] = {0, 1}; + + while (true) { + for (int i = 0; i < 2; i++) { + int err = cb(data, cwd[i]); + if (err) { + return err; + } + } + + int err = lfs_dir_fetch(lfs, &dir, cwd); + if (err) { + return err; + } + + // iterate over contents + while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { + int err = lfs_bd_read(lfs, dir.pair[0], dir.off, + &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + + dir.off += entry.d.len; + if ((0xf & entry.d.type) == (0xf & LFS_TYPE_REG)) { + int err = lfs_index_traverse(lfs, &lfs->rcache, NULL, + entry.d.u.file.head, entry.d.u.file.size, cb, data); + if (err) { + return err; + } + } + } + + cwd[0] = dir.d.tail[0]; + cwd[1] = dir.d.tail[1]; + + if (lfs_pairisnull(cwd)) { + break; + } + } + + // iterate over any open files + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (f->flags & LFS_F_DIRTY) { + int err = lfs_index_traverse(lfs, &lfs->rcache, &f->cache, + f->head, f->size, cb, data); + if (err) { + return err; + } + } + + if (f->flags & LFS_F_WRITING) { + int err = lfs_index_traverse(lfs, &lfs->rcache, &f->cache, + f->block, f->pos, cb, data); + if (err) { + return err; + } + } + } + + return 0; +} + +static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { + if (lfs_pairisnull(lfs->root)) { + return 0; + } + + // iterate over all directory directory entries + int err = lfs_dir_fetch(lfs, pdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + while (!lfs_pairisnull(pdir->d.tail)) { + if (lfs_paircmp(pdir->d.tail, dir) == 0) { + return true; + } + + int err = lfs_dir_fetch(lfs, pdir, pdir->d.tail); + if (err) { + return err; + } + } + + return false; +} + +static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], + lfs_dir_t *parent, lfs_entry_t *entry) { + if (lfs_pairisnull(lfs->root)) { + return 0; + } + + parent->d.tail[0] = 0; + parent->d.tail[1] = 1; + + // iterate over all directory directory entries + while (!lfs_pairisnull(parent->d.tail)) { + int err = lfs_dir_fetch(lfs, parent, parent->d.tail); + if (err) { + return err; + } + + while (true) { + int err = lfs_dir_next(lfs, parent, entry); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + if (err == LFS_ERR_NOENT) { + break; + } + + if (((0xf & entry->d.type) == (0xf & LFS_TYPE_DIR)) && + lfs_paircmp(entry->d.u.dir, dir) == 0) { + return true; + } + } + } + + return false; +} + +static int lfs_relocate(lfs_t *lfs, + const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { + // find parent + lfs_dir_t parent; + lfs_entry_t entry; + int res = lfs_parent(lfs, oldpair, &parent, &entry); + if (res < 0) { + return res; + } + + if (res) { + // update disk, this creates a desync + entry.d.u.dir[0] = newpair[0]; + entry.d.u.dir[1] = newpair[1]; + + int err = lfs_dir_update(lfs, &parent, &entry, NULL); + if (err) { + return err; + } + + // update internal root + if (lfs_paircmp(oldpair, lfs->root) == 0) { + LFS_DEBUG("Relocating root %d %d", newpair[0], newpair[1]); + lfs->root[0] = newpair[0]; + lfs->root[1] = newpair[1]; + } + + // clean up bad block, which should now be a desync + return lfs_deorphan(lfs); + } + + // find pred + res = lfs_pred(lfs, oldpair, &parent); + if (res < 0) { + return res; + } + + if (res) { + // just replace bad pair, no desync can occur + parent.d.tail[0] = newpair[0]; + parent.d.tail[0] = newpair[0]; + + return lfs_dir_commit(lfs, &parent, NULL, 0); + } + + // couldn't find dir, must be new + return 0; +} + +int lfs_deorphan(lfs_t *lfs) { + lfs->deorphaned = true; + if (lfs_pairisnull(lfs->root)) { + return 0; + } + + lfs_dir_t pdir; + lfs_dir_t cdir; + + // skip superblock + int err = lfs_dir_fetch(lfs, &pdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + // iterate over all directories + while (!lfs_pairisnull(pdir.d.tail)) { + int err = lfs_dir_fetch(lfs, &cdir, pdir.d.tail); + if (err) { + return err; + } + + // only check head blocks + if (!(0x80000000 & pdir.d.size)) { + // check if we have a parent + lfs_dir_t parent; + lfs_entry_t entry; + int res = lfs_parent(lfs, pdir.d.tail, &parent, &entry); + if (res < 0) { + return res; + } + + if (!res) { + // we are an orphan + LFS_DEBUG("Orphan %d %d", pdir.d.tail[0], pdir.d.tail[1]); + + pdir.d.tail[0] = cdir.d.tail[0]; + pdir.d.tail[1] = cdir.d.tail[1]; + + err = lfs_dir_commit(lfs, &pdir, NULL, 0); + if (err) { + return err; + } + + break; + } + + if (!lfs_pairsync(entry.d.u.dir, pdir.d.tail)) { + // we have desynced + LFS_DEBUG("Desync %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); + + pdir.d.tail[0] = entry.d.u.dir[0]; + pdir.d.tail[1] = entry.d.u.dir[1]; + + err = lfs_dir_commit(lfs, &pdir, NULL, 0); + if (err) { + return err; + } + + break; + } + } + + memcpy(&pdir, &cdir, sizeof(pdir)); + } + + return 0; +} + diff --git a/littlefs/lfs.h b/littlefs/lfs.h new file mode 100644 index 0000000000..0133b94aec --- /dev/null +++ b/littlefs/lfs.h @@ -0,0 +1,435 @@ +/* + * The little filesystem + * + * Copyright (c) 2017 Christopher Haster + * Distributed under the Apache 2.0 license + */ +#ifndef LFS_H +#define LFS_H + +#include +#include + + +/// Definitions /// + +// Type definitions +typedef uint32_t lfs_size_t; +typedef uint32_t lfs_off_t; + +typedef int32_t lfs_ssize_t; +typedef int32_t lfs_soff_t; + +typedef uint32_t lfs_block_t; + +// Max name size in bytes +#ifndef LFS_NAME_MAX +#define LFS_NAME_MAX 255 +#endif + +// Possible error codes, these are negative to allow +// valid positive return values +enum lfs_error { + LFS_ERR_OK = 0, // No error + LFS_ERR_IO = -5, // Error during device operation + LFS_ERR_CORRUPT = -52, // Corrupted + LFS_ERR_NOENT = -2, // No directory entry + LFS_ERR_EXISTS = -17, // Entry already exists + LFS_ERR_NOTDIR = -20, // Entry is not a dir + LFS_ERR_ISDIR = -21, // Entry is a dir + LFS_ERR_INVAL = -22, // Invalid parameter + LFS_ERR_NOSPC = -28, // No space left on device + LFS_ERR_NOMEM = -12, // No more memory available +}; + +// File types +enum lfs_type { + LFS_TYPE_REG = 0x11, + LFS_TYPE_DIR = 0x22, + LFS_TYPE_SUPERBLOCK = 0xe2, +}; + +// File open flags +enum lfs_open_flags { + // open flags + LFS_O_RDONLY = 1, // Open a file as read only + LFS_O_WRONLY = 2, // Open a file as write only + LFS_O_RDWR = 3, // Open a file as read and write + LFS_O_CREAT = 0x0100, // Create a file if it does not exist + LFS_O_EXCL = 0x0200, // Fail if a file already exists + LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size + LFS_O_APPEND = 0x0800, // Move to end of file on every write + + // internally used flags + LFS_F_DIRTY = 0x10000, // File does not match storage + LFS_F_WRITING = 0x20000, // File has been written since last flush + LFS_F_READING = 0x40000, // File has been read since last flush +}; + +// File seek flags +enum lfs_whence_flags { + LFS_SEEK_SET = 0, // Seek relative to an absolute position + LFS_SEEK_CUR = 1, // Seek relative to the current file position + LFS_SEEK_END = 2, // Seek relative to the end of the file +}; + + +// Configuration provided during initialization of the littlefs +struct lfs_config { + // Opaque user provided context that can be used to pass + // information to the block device operations + void *context; + + // Read a region in a block. Negative error codes are propogated + // to the user. + int (*read)(const struct lfs_config *c, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size); + + // Program a region in a block. The block must have previously + // been erased. Negative error codes are propogated to the user. + // The prog function must return LFS_ERR_CORRUPT if the block should + // be considered bad. + int (*prog)(const struct lfs_config *c, lfs_block_t block, + lfs_off_t off, const void *buffer, lfs_size_t size); + + // Erase a block. A block must be erased before being programmed. + // The state of an erased block is undefined. Negative error codes + // are propogated to the user. + int (*erase)(const struct lfs_config *c, lfs_block_t block); + + // Sync the state of the underlying block device. Negative error codes + // are propogated to the user. + int (*sync)(const struct lfs_config *c); + + // Minimum size of a block read. This determines the size of read buffers. + // This may be larger than the physical read size to improve performance + // by caching more of the block device. + lfs_size_t read_size; + + // Minimum size of a block program. This determines the size of program + // buffers. This may be larger than the physical program size to improve + // performance by caching more of the block device. + lfs_size_t prog_size; + + // Size of an erasable block. This does not impact ram consumption and + // may be larger than the physical erase size. However, this should be + // kept small as each file currently takes up an entire block . + lfs_size_t block_size; + + // Number of erasable blocks on the device. + lfs_size_t block_count; + + // Number of blocks to lookahead during block allocation. A larger + // lookahead reduces the number of passes required to allocate a block. + // The lookahead buffer requires only 1 bit per block so it can be quite + // large with little ram impact. Should be a multiple of 32. + lfs_size_t lookahead; + + // Optional, statically allocated read buffer. Must be read sized. + void *read_buffer; + + // Optional, statically allocated program buffer. Must be program sized. + void *prog_buffer; + + // Optional, statically allocated lookahead buffer. Must be 1 bit per + // lookahead block. + void *lookahead_buffer; + + // Optional, statically allocated buffer for files. Must be program sized. + // If enabled, only one file may be opened at a time. + void *file_buffer; +}; + + +// File info structure +struct lfs_info { + // Type of the file, either LFS_TYPE_REG or LFS_TYPE_DIR + uint8_t type; + + // Size of the file, only valid for REG files + lfs_size_t size; + + // Name of the file stored as a null-terminated string + char name[LFS_NAME_MAX+1]; +}; + + +/// littlefs data structures /// +typedef struct lfs_entry { + lfs_off_t off; + + struct lfs_disk_entry { + uint8_t type; + uint8_t name; + uint16_t len; + union { + struct { + lfs_block_t head; + lfs_size_t size; + } file; + lfs_block_t dir[2]; + } u; + } d; +} lfs_entry_t; + +typedef struct lfs_cache { + lfs_block_t block; + lfs_off_t off; + uint8_t *buffer; +} lfs_cache_t; + +typedef struct lfs_file { + struct lfs_file *next; + lfs_block_t pair[2]; + lfs_off_t poff; + + lfs_block_t head; + lfs_size_t size; + + uint32_t flags; + lfs_off_t pos; + lfs_block_t block; + lfs_off_t off; + lfs_cache_t cache; +} lfs_file_t; + +typedef struct lfs_dir { + lfs_block_t pair[2]; + lfs_off_t off; + + lfs_block_t head[2]; + lfs_off_t pos; + + struct lfs_disk_dir { + uint32_t rev; + lfs_size_t size; + lfs_block_t tail[2]; + } d; +} lfs_dir_t; + +typedef struct lfs_superblock { + lfs_off_t off; + + struct lfs_disk_superblock { + uint8_t type; + uint8_t name; + uint16_t len; + lfs_block_t root[2]; + uint32_t block_size; + uint32_t block_count; + uint32_t version; + char magic[8]; + } d; +} lfs_superblock_t; + +typedef struct lfs_free { + lfs_block_t end; + lfs_block_t start; + lfs_block_t off; + uint32_t *lookahead; +} lfs_free_t; + +// The littlefs type +typedef struct lfs { + const struct lfs_config *cfg; + + lfs_block_t root[2]; + lfs_file_t *files; + bool deorphaned; + + lfs_cache_t rcache; + lfs_cache_t pcache; + + lfs_free_t free; +} lfs_t; + + +/// Filesystem functions /// + +// Format a block device with the littlefs +// +// Requires a littlefs object and config struct. This clobbers the littlefs +// object, and does not leave the filesystem mounted. +// +// Returns a negative error code on failure. +int lfs_format(lfs_t *lfs, const struct lfs_config *config); + +// Mounts a littlefs +// +// Requires a littlefs object and config struct. Multiple filesystems +// may be mounted simultaneously with multiple littlefs objects. Both +// lfs and config must be allocated while mounted. +// +// Returns a negative error code on failure. +int lfs_mount(lfs_t *lfs, const struct lfs_config *config); + +// Unmounts a littlefs +// +// Does nothing besides releasing any allocated resources. +// Returns a negative error code on failure. +int lfs_unmount(lfs_t *lfs); + +/// General operations /// + +// Removes a file or directory +// +// If removing a directory, the directory must be empty. +// Returns a negative error code on failure. +int lfs_remove(lfs_t *lfs, const char *path); + +// Rename or move a file or directory +// +// If the destination exists, it must match the source in type. +// If the destination is a directory, the directory must be empty. +// +// Note: If power loss occurs, it is possible that the file or directory +// will exist in both the oldpath and newpath simultaneously after the +// next mount. +// +// Returns a negative error code on failure. +int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); + +// Find info about a file or directory +// +// Fills out the info structure, based on the specified file or directory. +// Returns a negative error code on failure. +int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); + + +/// File operations /// + +// Open a file +// +// The mode that the file is opened in is determined +// by the flags, which are values from the enum lfs_open_flags +// that are bitwise-ored together. +// +// Returns a negative error code on failure. +int lfs_file_open(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags); + +// Close a file +// +// Any pending writes are written out to storage as though +// sync had been called and releases any allocated resources. +// +// Returns a negative error code on failure. +int lfs_file_close(lfs_t *lfs, lfs_file_t *file); + +// Synchronize a file on storage +// +// Any pending writes are written out to storage. +// Returns a negative error code on failure. +int lfs_file_sync(lfs_t *lfs, lfs_file_t *file); + +// Read data from file +// +// Takes a buffer and size indicating where to store the read data. +// Returns the number of bytes read, or a negative error code on failure. +lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, + void *buffer, lfs_size_t size); + +// Write data to file +// +// Takes a buffer and size indicating the data to write. The file will not +// actually be updated on the storage until either sync or close is called. +// +// Returns the number of bytes written, or a negative error code on failure. +lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, + const void *buffer, lfs_size_t size); + +// Change the position of the file +// +// The change in position is determined by the offset and whence flag. +// Returns the old position of the file, or a negative error code on failure. +lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, + lfs_soff_t off, int whence); + +// Return the position of the file +// +// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_CUR) +// Returns the position of the file, or a negative error code on failure. +lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file); + +// Change the position of the file to the beginning of the file +// +// Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_CUR) +// Returns a negative error code on failure. +int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file); + +// Return the size of the file +// +// Similar to lfs_file_seek(lfs, file, 0, LFS_SEEK_END) +// Returns the size of the file, or a negative error code on failure. +lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); + + +/// Directory operations /// + +// Create a directory +// +// Returns a negative error code on failure. +int lfs_mkdir(lfs_t *lfs, const char *path); + +// Open a directory +// +// Once open a directory can be used with read to iterate over files. +// Returns a negative error code on failure. +int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path); + +// Close a directory +// +// Releases any allocated resources. +// Returns a negative error code on failure. +int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir); + +// Read an entry in the directory +// +// Fills out the info structure, based on the specified file or directory. +// Returns a negative error code on failure. +int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info); + +// Change the position of the directory +// +// The new off must be a value previous returned from tell and specifies +// an absolute offset in the directory seek. +// +// Returns a negative error code on failure. +int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off); + +// Return the position of the directory +// +// The returned offset is only meant to be consumed by seek and may not make +// sense, but does indicate the current position in the directory iteration. +// +// Returns the position of the directory, or a negative error code on failure. +lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir); + +// Change the position of the directory to the beginning of the directory +// +// Returns a negative error code on failure. +int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); + + +/// Miscellaneous littlefs specific operations /// + +// Traverse through all blocks in use by the filesystem +// +// The provided callback will be called with each block address that is +// currently in use by the filesystem. This can be used to determine which +// blocks are in use or how much of the storage is available. +// +// Returns a negative error code on failure. +int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); + +// Prunes any recoverable errors that may have occured in the filesystem +// +// Not needed to be called by user unless an operation is interrupted +// but the filesystem is still mounted. This is already called on first +// allocation. +// +// Returns a negative error code on failure. +int lfs_deorphan(lfs_t *lfs); + + +#endif diff --git a/littlefs/lfs_util.c b/littlefs/lfs_util.c new file mode 100644 index 0000000000..0a7234c3a5 --- /dev/null +++ b/littlefs/lfs_util.c @@ -0,0 +1,25 @@ +/* + * lfs util functions + * + * Copyright (c) 2017 Christopher Haster + * Distributed under the Apache 2.0 license + */ +#include "lfs_util.h" + + +void lfs_crc(uint32_t *restrict crc, const void *buffer, size_t size) { + static const uint32_t rtable[16] = { + 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, + 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, + 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, + 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c, + }; + + const uint8_t *data = buffer; + + for (size_t i = 0; i < size; i++) { + *crc = (*crc >> 4) ^ rtable[(*crc ^ (data[i] >> 0)) & 0xf]; + *crc = (*crc >> 4) ^ rtable[(*crc ^ (data[i] >> 4)) & 0xf]; + } +} + diff --git a/littlefs/lfs_util.h b/littlefs/lfs_util.h new file mode 100644 index 0000000000..95706d1c1d --- /dev/null +++ b/littlefs/lfs_util.h @@ -0,0 +1,45 @@ +/* + * lfs utility functions + * + * Copyright (c) 2017 Christopher Haster + * Distributed under the Apache 2.0 license + */ +#ifndef LFS_UTIL_H +#define LFS_UTIL_H + +#include +#include +#include + + +// Builtin functions +static inline uint32_t lfs_max(uint32_t a, uint32_t b) { + return (a > b) ? a : b; +} + +static inline uint32_t lfs_min(uint32_t a, uint32_t b) { + return (a < b) ? a : b; +} + +static inline uint32_t lfs_ctz(uint32_t a) { + return __builtin_ctz(a); +} + +static inline uint32_t lfs_npw2(uint32_t a) { + return 32 - __builtin_clz(a-1); +} + +static inline int lfs_scmp(uint32_t a, uint32_t b) { + return (int)(unsigned)(a - b); +} + +void lfs_crc(uint32_t *crc, const void *buffer, size_t size); + + +// Logging functions +#define LFS_DEBUG(fmt, ...) printf("lfs debug: " fmt "\n", __VA_ARGS__) +#define LFS_WARN(fmt, ...) printf("lfs warn: " fmt "\n", __VA_ARGS__) +#define LFS_ERROR(fmt, ...) printf("lfs error: " fmt "\n", __VA_ARGS__) + + +#endif diff --git a/littlefs/tests/stats.py b/littlefs/tests/stats.py new file mode 100755 index 0000000000..2ba1fb654c --- /dev/null +++ b/littlefs/tests/stats.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import struct +import sys +import time +import os +import re + +def main(): + with open('blocks/config') as file: + s = struct.unpack(' +#include +#include + + +// test stuff +void test_log(const char *s, uintmax_t v) {{ + printf("%s: %jd\n", s, v); +}} + +void test_assert(const char *file, unsigned line, + const char *s, uintmax_t v, uintmax_t e) {{ + static const char *last[6] = {{0, 0}}; + if (v != e || !(last[0] == s || last[1] == s || + last[2] == s || last[3] == s || + last[4] == s || last[5] == s)) {{ + test_log(s, v); + last[0] = last[1]; + last[1] = last[2]; + last[2] = last[3]; + last[3] = last[4]; + last[4] = last[5]; + last[5] = s; + }} + + if (v != e) {{ + printf("\033[31m%s:%u: assert %s failed, expected %jd\033[0m\n", + file, line, s, e); + exit(-2); + }} +}} + +#define test_assert(s, v, e) test_assert(__FILE__, __LINE__, s, v, e) + + +// utility functions for traversals +int test_count(void *p, lfs_block_t b) {{ + unsigned *u = (unsigned*)p; + *u += 1; + return 0; +}} + + +// lfs declarations +lfs_t lfs; +lfs_emubd_t bd; +lfs_file_t file[4]; +lfs_dir_t dir[4]; +struct lfs_info info; + +uint8_t buffer[1024]; +uint8_t wbuffer[1024]; +uint8_t rbuffer[1024]; +lfs_size_t size; +lfs_size_t wsize; +lfs_size_t rsize; + +uintmax_t res; + +#ifndef LFS_READ_SIZE +#define LFS_READ_SIZE 16 +#endif + +#ifndef LFS_PROG_SIZE +#define LFS_PROG_SIZE 16 +#endif + +#ifndef LFS_BLOCK_SIZE +#define LFS_BLOCK_SIZE 512 +#endif + +#ifndef LFS_BLOCK_COUNT +#define LFS_BLOCK_COUNT 1024 +#endif + +#ifndef LFS_LOOKAHEAD +#define LFS_LOOKAHEAD 128 +#endif + +const struct lfs_config cfg = {{ + .context = &bd, + .read = &lfs_emubd_read, + .prog = &lfs_emubd_prog, + .erase = &lfs_emubd_erase, + .sync = &lfs_emubd_sync, + + .read_size = LFS_READ_SIZE, + .prog_size = LFS_PROG_SIZE, + .block_size = LFS_BLOCK_SIZE, + .block_count = LFS_BLOCK_COUNT, + .lookahead = LFS_LOOKAHEAD, +}}; + + +// Entry point +int main() {{ + lfs_emubd_create(&cfg, "blocks"); + +{tests} + + lfs_emubd_destroy(&cfg); +}} diff --git a/littlefs/tests/test.py b/littlefs/tests/test.py new file mode 100755 index 0000000000..aa125c7c2f --- /dev/null +++ b/littlefs/tests/test.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +import re +import sys +import subprocess +import os + +def generate(test): + with open("tests/template.fmt") as file: + template = file.read() + + lines = [] + for line in re.split('(?<=[;{}])\n', test.read()): + match = re.match('(?: *\n)*( *)(.*)=>(.*);', line, re.DOTALL | re.MULTILINE) + if match: + tab, test, expect = match.groups() + lines.append(tab+'res = {test};'.format(test=test.strip())) + lines.append(tab+'test_assert("{name}", res, {expect});'.format( + name = re.match('\w*', test.strip()).group(), + expect = expect.strip())) + else: + lines.append(line) + + with open('test.c', 'w') as file: + file.write(template.format(tests='\n'.join(lines))) + +def compile(): + os.environ['CFLAGS'] = os.environ.get('CFLAGS', '') + ' -Werror' + subprocess.check_call(['make', '--no-print-directory', '-s'], env=os.environ) + +def execute(): + subprocess.check_call(["./lfs"]) + +def main(test=None): + if test and not test.startswith('-'): + with open(test) as file: + generate(file) + else: + generate(sys.stdin) + + compile() + + if test == '-s': + sys.exit(1) + + execute() + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/littlefs/tests/test_alloc.sh b/littlefs/tests/test_alloc.sh new file mode 100755 index 0000000000..630be2a1bc --- /dev/null +++ b/littlefs/tests/test_alloc.sh @@ -0,0 +1,261 @@ +#!/bin/bash +set -eu + +echo "=== Allocator tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +SIZE=15000 + +lfs_mkdir() { +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "$1") => 0; + lfs_unmount(&lfs) => 0; +TEST +} + +lfs_remove() { +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_remove(&lfs, "$1/eggs") => 0; + lfs_remove(&lfs, "$1/bacon") => 0; + lfs_remove(&lfs, "$1/pancakes") => 0; + lfs_remove(&lfs, "$1") => 0; + lfs_unmount(&lfs) => 0; +TEST +} + +lfs_alloc_singleproc() { +tests/test.py << TEST + const char *names[] = {"bacon", "eggs", "pancakes"}; + lfs_mount(&lfs, &cfg) => 0; + for (int n = 0; n < sizeof(names)/sizeof(names[0]); n++) { + sprintf((char*)buffer, "$1/%s", names[n]); + lfs_file_open(&lfs, &file[n], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; + } + for (int n = 0; n < sizeof(names)/sizeof(names[0]); n++) { + size = strlen(names[n]); + for (int i = 0; i < $SIZE; i++) { + lfs_file_write(&lfs, &file[n], names[n], size) => size; + } + } + for (int n = 0; n < sizeof(names)/sizeof(names[0]); n++) { + lfs_file_close(&lfs, &file[n]) => 0; + } + lfs_unmount(&lfs) => 0; +TEST +} + +lfs_alloc_multiproc() { +for name in bacon eggs pancakes +do +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "$1/$name", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; + size = strlen("$name"); + memcpy(buffer, "$name", size); + for (int i = 0; i < $SIZE; i++) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +done +} + +lfs_verify() { +for name in bacon eggs pancakes +do +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "$1/$name", LFS_O_RDONLY) => 0; + size = strlen("$name"); + for (int i = 0; i < $SIZE; i++) { + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "$name", size) => 0; + } + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +done +} + +echo "--- Single-process allocation test ---" +lfs_mkdir singleproc +lfs_alloc_singleproc singleproc +lfs_verify singleproc + +echo "--- Multi-process allocation test ---" +lfs_mkdir multiproc +lfs_alloc_multiproc multiproc +lfs_verify multiproc +lfs_verify singleproc + +echo "--- Single-process reuse test ---" +lfs_remove singleproc +lfs_mkdir singleprocreuse +lfs_alloc_singleproc singleprocreuse +lfs_verify singleprocreuse +lfs_verify multiproc + +echo "--- Multi-process reuse test ---" +lfs_remove multiproc +lfs_mkdir multiprocreuse +lfs_alloc_singleproc multiprocreuse +lfs_verify multiprocreuse +lfs_verify singleprocreuse + +echo "--- Cleanup ---" +lfs_remove multiprocreuse +lfs_remove singleprocreuse + +echo "--- Exhaustion test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + size = strlen("exhaustion"); + memcpy(buffer, "exhaustion", size); + lfs_file_write(&lfs, &file[0], buffer, size) => size; + + size = strlen("blahblahblahblah"); + memcpy(buffer, "blahblahblahblah", size); + lfs_ssize_t res; + while (true) { + res = lfs_file_write(&lfs, &file[0], buffer, size); + if (res < 0) { + break; + } + + res => size; + } + res => LFS_ERR_NOSPC; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_RDONLY); + size = strlen("exhaustion"); + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "exhaustion", size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Exhaustion wraparound test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_remove(&lfs, "exhaustion") => 0; + + lfs_file_open(&lfs, &file[0], "padding", LFS_O_WRONLY | LFS_O_CREAT); + size = strlen("buffering"); + memcpy(buffer, "buffering", size); + for (int i = 0; i < $SIZE; i++) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + lfs_file_close(&lfs, &file[0]) => 0; + lfs_remove(&lfs, "padding") => 0; + + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + size = strlen("exhaustion"); + memcpy(buffer, "exhaustion", size); + lfs_file_write(&lfs, &file[0], buffer, size) => size; + + size = strlen("blahblahblahblah"); + memcpy(buffer, "blahblahblahblah", size); + lfs_ssize_t res; + while (true) { + res = lfs_file_write(&lfs, &file[0], buffer, size); + if (res < 0) { + break; + } + + res => size; + } + res => LFS_ERR_NOSPC; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_RDONLY); + size = strlen("exhaustion"); + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "exhaustion", size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Dir exhaustion test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "exhaustion", &info) => 0; + lfs_size_t fullsize = info.size; + lfs_remove(&lfs, "exhaustion") => 0; + + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + size = strlen("blahblahblahblah"); + memcpy(buffer, "blahblahblahblah", size); + for (lfs_size_t i = 0; i < fullsize - 2*512; i += size) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + lfs_file_close(&lfs, &file[0]) => 0; + + lfs_mkdir(&lfs, "exhaustiondir") => 0; + lfs_remove(&lfs, "exhaustiondir") => 0; + + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_APPEND); + size = strlen("blahblahblahblah"); + memcpy(buffer, "blahblahblahblah", size); + lfs_file_write(&lfs, &file[0], buffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + lfs_mkdir(&lfs, "exhaustiondir") => LFS_ERR_NOSPC; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Chained dir exhaustion test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "exhaustion", &info) => 0; + lfs_size_t fullsize = info.size; + + lfs_remove(&lfs, "exhaustion") => 0; + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + size = strlen("blahblahblahblah"); + memcpy(buffer, "blahblahblahblah", size); + for (lfs_size_t i = 0; i < fullsize - 19*512; i += size) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + lfs_file_close(&lfs, &file[0]) => 0; + + for (int i = 0; i < 9; i++) { + sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); + lfs_mkdir(&lfs, (char*)buffer) => 0; + } + + lfs_mkdir(&lfs, "exhaustiondir") => LFS_ERR_NOSPC; + + lfs_remove(&lfs, "exhaustion") => 0; + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + size = strlen("blahblahblahblah"); + memcpy(buffer, "blahblahblahblah", size); + for (lfs_size_t i = 0; i < fullsize - 20*512; i += size) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + lfs_file_close(&lfs, &file[0]) => 0; + + lfs_mkdir(&lfs, "exhaustiondir") => 0; + lfs_mkdir(&lfs, "exhaustiondir2") => LFS_ERR_NOSPC; +TEST + + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_corrupt.sh b/littlefs/tests/test_corrupt.sh new file mode 100755 index 0000000000..d79a8c8964 --- /dev/null +++ b/littlefs/tests/test_corrupt.sh @@ -0,0 +1,106 @@ +#!/bin/bash +set -eu + +echo "=== Corrupt tests ===" + +NAMEMULT=64 +FILEMULT=1 + +lfs_mktree() { +tests/test.py ${1:-} << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + for (int i = 1; i < 10; i++) { + for (int j = 0; j < $NAMEMULT; j++) { + buffer[j] = '0'+i; + } + buffer[$NAMEMULT] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => 0; + + buffer[$NAMEMULT] = '/'; + for (int j = 0; j < $NAMEMULT; j++) { + buffer[j+$NAMEMULT+1] = '0'+i; + } + buffer[2*$NAMEMULT+1] = '\0'; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => 0; + + size = $NAMEMULT; + for (int j = 0; j < i*$FILEMULT; j++) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + + lfs_file_close(&lfs, &file[0]) => 0; + } + lfs_unmount(&lfs) => 0; +TEST +} + +lfs_chktree() { +tests/test.py ${1:-} << TEST + lfs_mount(&lfs, &cfg) => 0; + for (int i = 1; i < 10; i++) { + for (int j = 0; j < $NAMEMULT; j++) { + buffer[j] = '0'+i; + } + buffer[$NAMEMULT] = '\0'; + lfs_stat(&lfs, (char*)buffer, &info) => 0; + info.type => LFS_TYPE_DIR; + + buffer[$NAMEMULT] = '/'; + for (int j = 0; j < $NAMEMULT; j++) { + buffer[j+$NAMEMULT+1] = '0'+i; + } + buffer[2*$NAMEMULT+1] = '\0'; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + + size = $NAMEMULT; + for (int j = 0; j < i*$FILEMULT; j++) { + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(buffer, rbuffer, size) => 0; + } + + lfs_file_close(&lfs, &file[0]) => 0; + } + lfs_unmount(&lfs) => 0; +TEST +} + +echo "--- Sanity check ---" +rm -rf blocks +lfs_mktree +lfs_chktree + +echo "--- Block corruption ---" +for i in {0..33} +do + rm -rf blocks + mkdir blocks + ln -s /dev/zero blocks/$(printf '%x' $i) + lfs_mktree + lfs_chktree +done + +echo "--- Big region corruption ---" +rm -rf blocks +mkdir blocks +for i in {2..255} +do + ln -s /dev/zero blocks/$(printf '%x' $i) +done +lfs_mktree +lfs_chktree + +echo "--- Alternating corruption ---" +rm -rf blocks +mkdir blocks +for i in {2..511..2} +do + ln -s /dev/zero blocks/$(printf '%x' $i) +done +lfs_mktree +lfs_chktree + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_dirs.sh b/littlefs/tests/test_dirs.sh new file mode 100755 index 0000000000..815b88bd22 --- /dev/null +++ b/littlefs/tests/test_dirs.sh @@ -0,0 +1,287 @@ +#!/bin/bash +set -eu + +LARGESIZE=128 + +echo "=== Directory tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +echo "--- Root directory ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Directory creation ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "potato") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- File creation ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "burito", LFS_O_CREAT | LFS_O_WRONLY) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Directory iteration ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "potato") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "burito") => 0; + info.type => LFS_TYPE_REG; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Directory failures ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "potato") => LFS_ERR_EXISTS; + lfs_dir_open(&lfs, &dir[0], "tomato") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "burito") => LFS_ERR_NOTDIR; + lfs_file_open(&lfs, &file[0], "tomato", LFS_O_RDONLY) => LFS_ERR_NOENT; + lfs_file_open(&lfs, &file[0], "potato", LFS_O_RDONLY) => LFS_ERR_ISDIR; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Nested directories ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "potato/baked") => 0; + lfs_mkdir(&lfs, "potato/sweet") => 0; + lfs_mkdir(&lfs, "potato/fried") => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "potato") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "baked") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "sweet") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "fried") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Multi-block directory ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "cactus") => 0; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "cactus/test%d", i); + lfs_mkdir(&lfs, (char*)buffer) => 0; + } + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "cactus") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "test%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + } + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Directory remove ---" +# TESTING HERE +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_remove(&lfs, "potato") => LFS_ERR_INVAL; + lfs_remove(&lfs, "potato/sweet") => 0; + lfs_remove(&lfs, "potato/baked") => 0; + lfs_remove(&lfs, "potato/fried") => 0; + + lfs_dir_open(&lfs, &dir[0], "potato") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + + lfs_remove(&lfs, "potato") => 0; + + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "burito") => 0; + info.type => LFS_TYPE_REG; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "cactus") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "burito") => 0; + info.type => LFS_TYPE_REG; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "cactus") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Directory rename ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "coldpotato") => 0; + lfs_mkdir(&lfs, "coldpotato/baked") => 0; + lfs_mkdir(&lfs, "coldpotato/sweet") => 0; + lfs_mkdir(&lfs, "coldpotato/fried") => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_rename(&lfs, "coldpotato", "hotpotato") => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "hotpotato") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "baked") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "sweet") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "fried") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "warmpotato") => 0; + lfs_mkdir(&lfs, "warmpotato/mushy") => 0; + lfs_rename(&lfs, "hotpotato", "warmpotato") => LFS_ERR_INVAL; + + lfs_remove(&lfs, "warmpotato/mushy") => 0; + lfs_rename(&lfs, "hotpotato", "warmpotato") => 0; + + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "warmpotato") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "baked") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "sweet") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "fried") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "coldpotato") => 0; + lfs_rename(&lfs, "warmpotato/baked", "coldpotato/baked") => 0; + lfs_rename(&lfs, "warmpotato/sweet", "coldpotato/sweet") => 0; + lfs_rename(&lfs, "warmpotato/fried", "coldpotato/fried") => 0; + lfs_remove(&lfs, "coldpotato") => LFS_ERR_INVAL; + lfs_remove(&lfs, "warmpotato") => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "coldpotato") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "baked") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "sweet") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "fried") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_files.sh b/littlefs/tests/test_files.sh new file mode 100755 index 0000000000..6086107c89 --- /dev/null +++ b/littlefs/tests/test_files.sh @@ -0,0 +1,114 @@ +#!/bin/bash +set -eu + +SMALLSIZE=32 +MEDIUMSIZE=8192 +LARGESIZE=262144 + +echo "=== File tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +echo "--- Simple file test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello", LFS_O_WRONLY | LFS_O_CREAT) => 0; + size = strlen("Hello World!\n"); + memcpy(wbuffer, "Hello World!\n", size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + lfs_file_open(&lfs, &file[0], "hello", LFS_O_RDONLY) => 0; + size = strlen("Hello World!\n"); + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +w_test() { +tests/test.py << TEST + lfs_size_t size = $1; + lfs_size_t chunk = 31; + srand(0); + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "$2", LFS_O_WRONLY | LFS_O_CREAT) => 0; + for (lfs_size_t i = 0; i < size; i += chunk) { + chunk = (chunk < size - i) ? chunk : size - i; + for (lfs_size_t b = 0; b < chunk; b++) { + buffer[b] = rand() & 0xff; + } + lfs_file_write(&lfs, &file[0], buffer, chunk) => chunk; + } + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +} + +r_test() { +tests/test.py << TEST + lfs_size_t size = $1; + lfs_size_t chunk = 29; + srand(0); + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "$2", LFS_O_RDONLY) => 0; + for (lfs_size_t i = 0; i < size; i += chunk) { + chunk = (chunk < size - i) ? chunk : size - i; + lfs_file_read(&lfs, &file[0], buffer, chunk) => chunk; + for (lfs_size_t b = 0; b < chunk && i+b < size; b++) { + buffer[b] => rand() & 0xff; + } + } + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +} + +echo "--- Small file test ---" +w_test $SMALLSIZE smallavacado +r_test $SMALLSIZE smallavacado + +echo "--- Medium file test ---" +w_test $MEDIUMSIZE mediumavacado +r_test $MEDIUMSIZE mediumavacado + +echo "--- Large file test ---" +w_test $LARGESIZE largeavacado +r_test $LARGESIZE largeavacado + +echo "--- Non-overlap check ---" +r_test $SMALLSIZE smallavacado +r_test $MEDIUMSIZE mediumavacado +r_test $LARGESIZE largeavacado + +echo "--- Dir check ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hello") => 0; + info.type => LFS_TYPE_REG; + info.size => strlen("Hello World!\n"); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "smallavacado") => 0; + info.type => LFS_TYPE_REG; + info.size => $SMALLSIZE; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "mediumavacado") => 0; + info.type => LFS_TYPE_REG; + info.size => $MEDIUMSIZE; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "largeavacado") => 0; + info.type => LFS_TYPE_REG; + info.size => $LARGESIZE; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_format.sh b/littlefs/tests/test_format.sh new file mode 100755 index 0000000000..b9071015d6 --- /dev/null +++ b/littlefs/tests/test_format.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -eu + +echo "=== Formatting tests ===" +rm -rf blocks + +echo "--- Basic formatting ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +echo "--- Invalid superblocks ---" +ln -f -s /dev/zero blocks/0 +ln -f -s /dev/zero blocks/1 +tests/test.py << TEST + lfs_format(&lfs, &cfg) => LFS_ERR_CORRUPT; +TEST +rm blocks/0 blocks/1 + +echo "--- Basic mounting ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Invalid mount ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST +rm blocks/0 blocks/1 +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; +TEST + +echo "--- Valid corrupt mount ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST +rm blocks/0 +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_orphan.sh b/littlefs/tests/test_orphan.sh new file mode 100755 index 0000000000..71d6d4fc09 --- /dev/null +++ b/littlefs/tests/test_orphan.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -eu + +echo "=== Orphan tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +echo "--- Orphan test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "parent") => 0; + lfs_mkdir(&lfs, "parent/orphan") => 0; + lfs_mkdir(&lfs, "parent/child") => 0; + lfs_remove(&lfs, "parent/orphan") => 0; +TEST +# remove most recent file, this should be the update to the previous +# linked-list entry and should orphan the child +rm -v blocks/8 +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; + unsigned before = 0; + lfs_traverse(&lfs, test_count, &before) => 0; + test_log("before", before); + + lfs_deorphan(&lfs) => 0; + + lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; + unsigned after = 0; + lfs_traverse(&lfs, test_count, &after) => 0; + test_log("after", after); + + int diff = before - after; + diff => 2; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_parallel.sh b/littlefs/tests/test_parallel.sh new file mode 100755 index 0000000000..71c9c1f3b9 --- /dev/null +++ b/littlefs/tests/test_parallel.sh @@ -0,0 +1,186 @@ +#!/bin/bash +set -eu + +echo "=== Parallel tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +echo "--- Parallel file test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "a", LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_open(&lfs, &file[1], "b", LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_open(&lfs, &file[2], "c", LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_open(&lfs, &file[3], "d", LFS_O_WRONLY | LFS_O_CREAT) => 0; + + for (int i = 0; i < 10; i++) { + lfs_file_write(&lfs, &file[0], (const void*)"a", 1) => 1; + lfs_file_write(&lfs, &file[1], (const void*)"b", 1) => 1; + lfs_file_write(&lfs, &file[2], (const void*)"c", 1) => 1; + lfs_file_write(&lfs, &file[3], (const void*)"d", 1) => 1; + } + + lfs_file_close(&lfs, &file[0]); + lfs_file_close(&lfs, &file[1]); + lfs_file_close(&lfs, &file[2]); + lfs_file_close(&lfs, &file[3]); + + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "a") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "b") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "c") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "d") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + + lfs_file_open(&lfs, &file[0], "a", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[1], "b", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[2], "c", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[3], "d", LFS_O_RDONLY) => 0; + + for (int i = 0; i < 10; i++) { + lfs_file_read(&lfs, &file[0], buffer, 1) => 1; + buffer[0] => 'a'; + lfs_file_read(&lfs, &file[1], buffer, 1) => 1; + buffer[0] => 'b'; + lfs_file_read(&lfs, &file[2], buffer, 1) => 1; + buffer[0] => 'c'; + lfs_file_read(&lfs, &file[3], buffer, 1) => 1; + buffer[0] => 'd'; + } + + lfs_file_close(&lfs, &file[0]); + lfs_file_close(&lfs, &file[1]); + lfs_file_close(&lfs, &file[2]); + lfs_file_close(&lfs, &file[3]); + + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Parallel remove file test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "e", LFS_O_WRONLY | LFS_O_CREAT) => 0; + + for (int i = 0; i < 5; i++) { + lfs_file_write(&lfs, &file[0], (const void*)"e", 1) => 1; + } + + lfs_remove(&lfs, "a") => 0; + lfs_remove(&lfs, "b") => 0; + lfs_remove(&lfs, "c") => 0; + lfs_remove(&lfs, "d") => 0; + + for (int i = 0; i < 5; i++) { + lfs_file_write(&lfs, &file[0], (const void*)"e", 1) => 1; + } + + lfs_file_close(&lfs, &file[0]); + + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "e") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + + lfs_file_open(&lfs, &file[0], "e", LFS_O_RDONLY) => 0; + + for (int i = 0; i < 10; i++) { + lfs_file_read(&lfs, &file[0], buffer, 1) => 1; + buffer[0] => 'e'; + } + + lfs_file_close(&lfs, &file[0]); + + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Remove inconveniently test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "e", LFS_O_WRONLY | LFS_O_TRUNC) => 0; + lfs_file_open(&lfs, &file[1], "f", LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_open(&lfs, &file[2], "g", LFS_O_WRONLY | LFS_O_CREAT) => 0; + + for (int i = 0; i < 5; i++) { + lfs_file_write(&lfs, &file[0], (const void*)"e", 1) => 1; + lfs_file_write(&lfs, &file[1], (const void*)"f", 1) => 1; + lfs_file_write(&lfs, &file[2], (const void*)"g", 1) => 1; + } + + lfs_remove(&lfs, "f") => 0; + + for (int i = 0; i < 5; i++) { + lfs_file_write(&lfs, &file[0], (const void*)"e", 1) => 1; + lfs_file_write(&lfs, &file[1], (const void*)"f", 1) => 1; + lfs_file_write(&lfs, &file[2], (const void*)"g", 1) => 1; + } + + lfs_file_close(&lfs, &file[0]); + lfs_file_close(&lfs, &file[1]); + lfs_file_close(&lfs, &file[2]); + + lfs_dir_open(&lfs, &dir[0], "/") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "e") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "g") => 0; + info.type => LFS_TYPE_REG; + info.size => 10; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + + lfs_file_open(&lfs, &file[0], "e", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[1], "g", LFS_O_RDONLY) => 0; + + for (int i = 0; i < 10; i++) { + lfs_file_read(&lfs, &file[0], buffer, 1) => 1; + buffer[0] => 'e'; + lfs_file_read(&lfs, &file[1], buffer, 1) => 1; + buffer[0] => 'g'; + } + + lfs_file_close(&lfs, &file[0]); + lfs_file_close(&lfs, &file[1]); + + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_paths.sh b/littlefs/tests/test_paths.sh new file mode 100755 index 0000000000..769f37fcea --- /dev/null +++ b/littlefs/tests/test_paths.sh @@ -0,0 +1,86 @@ +#!/bin/bash +set -eu + +echo "=== Path tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST + +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "tea") => 0; + lfs_mkdir(&lfs, "coffee") => 0; + lfs_mkdir(&lfs, "soda") => 0; + lfs_mkdir(&lfs, "tea/hottea") => 0; + lfs_mkdir(&lfs, "tea/warmtea") => 0; + lfs_mkdir(&lfs, "tea/coldtea") => 0; + lfs_mkdir(&lfs, "coffee/hotcoffee") => 0; + lfs_mkdir(&lfs, "coffee/warmcoffee") => 0; + lfs_mkdir(&lfs, "coffee/coldcoffee") => 0; + lfs_mkdir(&lfs, "soda/hotsoda") => 0; + lfs_mkdir(&lfs, "soda/warmsoda") => 0; + lfs_mkdir(&lfs, "soda/coldsoda") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Root path tests ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "/tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Redundant slash path tests ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "/tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "//tea//hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "///tea///hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Dot path tests ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "./tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "/./tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "/././tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "/./tea/./hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Dot dot path tests ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "coffee/../tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "tea/coldtea/../hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "coffee/coldcoffee/../../tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "coffee/../soda/../tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Root dot dot path tests ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "coffee/../../../../../../tea/hottea", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/littlefs/tests/test_seek.sh b/littlefs/tests/test_seek.sh new file mode 100755 index 0000000000..8c0093852e --- /dev/null +++ b/littlefs/tests/test_seek.sh @@ -0,0 +1,281 @@ +#!/bin/bash +set -eu + +SMALLSIZE=4 +MEDIUMSIZE=128 +LARGESIZE=132 + +echo "=== Seek tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "hello") => 0; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "hello/kitty%d", i); + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; + + size = strlen("kittycatcat"); + memcpy(buffer, "kittycatcat", size); + for (int j = 0; j < $LARGESIZE; j++) { + lfs_file_write(&lfs, &file[0], buffer, size); + } + + lfs_file_close(&lfs, &file[0]) => 0; + } + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Simple dir seek ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "hello") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + + lfs_soff_t pos; + int i; + for (i = 0; i < $SMALLSIZE; i++) { + sprintf((char*)buffer, "kitty%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + pos = lfs_dir_tell(&lfs, &dir[0]); + } + pos >= 0 => 1; + + lfs_dir_seek(&lfs, &dir[0], pos) => 0; + sprintf((char*)buffer, "kitty%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + + lfs_dir_rewind(&lfs, &dir[0]) => 0; + sprintf((char*)buffer, "kitty%d", 0); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + + lfs_dir_seek(&lfs, &dir[0], pos) => 0; + sprintf((char*)buffer, "kitty%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Large dir seek ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "hello") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + + lfs_soff_t pos; + int i; + for (i = 0; i < $MEDIUMSIZE; i++) { + sprintf((char*)buffer, "kitty%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + pos = lfs_dir_tell(&lfs, &dir[0]); + } + pos >= 0 => 1; + + lfs_dir_seek(&lfs, &dir[0], pos) => 0; + sprintf((char*)buffer, "kitty%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + + lfs_dir_rewind(&lfs, &dir[0]) => 0; + sprintf((char*)buffer, "kitty%d", 0); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + + lfs_dir_seek(&lfs, &dir[0], pos) => 0; + sprintf((char*)buffer, "kitty%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Simple file seek ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDONLY) => 0; + + lfs_soff_t pos; + size = strlen("kittycatcat"); + for (int i = 0; i < $SMALLSIZE; i++) { + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + pos = lfs_file_tell(&lfs, &file[0]); + } + pos >= 0 => 1; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => pos; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_rewind(&lfs, &file[0]) => 0; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], -size, LFS_SEEK_CUR) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], -size, LFS_SEEK_END) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_size_t size = lfs_file_size(&lfs, &file[0]); + lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_CUR) => size; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Large file seek ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDONLY) => 0; + + lfs_soff_t pos; + size = strlen("kittycatcat"); + for (int i = 0; i < $MEDIUMSIZE; i++) { + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + pos = lfs_file_tell(&lfs, &file[0]); + } + pos >= 0 => 1; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => pos; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_rewind(&lfs, &file[0]) => 0; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], -size, LFS_SEEK_CUR) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], -size, LFS_SEEK_END) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_size_t size = lfs_file_size(&lfs, &file[0]); + lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_CUR) => size; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Simple file seek and write ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDWR) => 0; + + lfs_soff_t pos; + size = strlen("kittycatcat"); + for (int i = 0; i < $SMALLSIZE; i++) { + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + pos = lfs_file_tell(&lfs, &file[0]); + } + pos >= 0 => 1; + + memcpy(buffer, "doggodogdog", size); + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => pos; + lfs_file_write(&lfs, &file[0], buffer, size) => size; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "doggodogdog", size) => 0; + + lfs_file_rewind(&lfs, &file[0]) => 0; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "doggodogdog", size) => 0; + + lfs_file_seek(&lfs, &file[0], -size, LFS_SEEK_END) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_size_t size = lfs_file_size(&lfs, &file[0]); + lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_CUR) => size; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Large file seek and write ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDWR) => 0; + + lfs_soff_t pos; + size = strlen("kittycatcat"); + for (int i = 0; i < $MEDIUMSIZE; i++) { + lfs_file_read(&lfs, &file[0], buffer, size) => size; + if (i != $SMALLSIZE) { + memcmp(buffer, "kittycatcat", size) => 0; + } + pos = lfs_file_tell(&lfs, &file[0]); + } + pos >= 0 => 1; + + memcpy(buffer, "doggodogdog", size); + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => pos; + lfs_file_write(&lfs, &file[0], buffer, size) => size; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "doggodogdog", size) => 0; + + lfs_file_rewind(&lfs, &file[0]) => 0; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_file_seek(&lfs, &file[0], pos, LFS_SEEK_SET) => size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "doggodogdog", size) => 0; + + lfs_file_seek(&lfs, &file[0], -size, LFS_SEEK_END) => pos+size; + lfs_file_read(&lfs, &file[0], buffer, size) => size; + memcmp(buffer, "kittycatcat", size) => 0; + + lfs_size_t size = lfs_file_size(&lfs, &file[0]); + lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_CUR) => size; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py