TheCloud/f3
1
0
Fork 0
mirror of https://github.com/AltraMayor/f3.git synced 2025-08-03 18:46:00 -04:00
Code Issues Packages Projects Releases Wiki Activity

f3probe: deal with permanent cache

Browse Source 
This patch addresses issue discussed here:
https://github.com/AltraMayor/f3/issues/15

Given the structure of the solution implemented in this patch,
it's also expected to address the following issue:
https://github.com/AltraMayor/f3/issues/16
This commit is contained in:
Michel Machado 2015-11-03 16:45:53 -05:00
parent 3f0efeb12f
commit 5d76cd84b6
6 changed files with 655 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
f3probe.c
View File

@ -254,14 +254,18 @@ static int unit_test(const char *filename)
enum fake_type origin_type = dev_param_to_type( enum fake_type origin_type = dev_param_to_type(
item->real_size_byte, item->fake_size_byte, item->real_size_byte, item->fake_size_byte,
item->wrap, item->block_order); item->wrap, item->block_order);
uint64_t item_cache_byte = item->cache_order < 0 ? 0 :
1ULL << (item->cache_order + item->block_order);
double f_real = item->real_size_byte; double f_real = item->real_size_byte;
double f_fake = item->fake_size_byte; double f_fake = item->fake_size_byte;
double f_cache = item_cache_byte;
const char *unit_real = adjust_unit(&f_real); const char *unit_real = adjust_unit(&f_real);
const char *unit_fake = adjust_unit(&f_fake); const char *unit_fake = adjust_unit(&f_fake);
const char *unit_cache = adjust_unit(&f_cache);
enum fake_type fake_type; enum fake_type fake_type;
uint64_t real_size_byte, announced_size_byte; uint64_t real_size_byte, announced_size_byte, cache_size_block;
int wrap, block_order, max_probe_blocks; int wrap, need_reset, block_order, max_probe_blocks;
struct device *dev; struct device *dev;
dev = create_file_device(filename, item->real_size_byte, dev = create_file_device(filename, item->real_size_byte,
@ -270,21 +274,23 @@ static int unit_test(const char *filename)
assert(dev); assert(dev);
max_probe_blocks = probe_device_max_blocks(dev); max_probe_blocks = probe_device_max_blocks(dev);
assert(!probe_device(dev, &real_size_byte, &announced_size_byte, assert(!probe_device(dev, &real_size_byte, &announced_size_byte,
&wrap, &block_order)); &wrap, &cache_size_block, &need_reset, &block_order));
free_device(dev); free_device(dev);
fake_type = dev_param_to_type(real_size_byte, fake_type = dev_param_to_type(real_size_byte,
announced_size_byte, wrap, block_order); announced_size_byte, wrap, block_order);
/* Report */ /* Report */
printf("Test %i\t\ttype/real size/fake size/module/block size\n", printf("Test %i\t\ttype/real size/fake size/module/cache size/reset/block size\n",
i + 1); i + 1);
printf("\t\t%s/%.2f %s/%.2f %s/2^%i Byte/2^%i Byte\n", printf("\t\t%s/%.2f %s/%.2f %s/2^%i Byte/%.2f %s/no/2^%i Byte\n",
fake_type_to_name(origin_type), fake_type_to_name(origin_type),
f_real, unit_real, f_fake, unit_fake, item->wrap, f_real, unit_real, f_fake, unit_fake, item->wrap,
item->block_order); f_cache, unit_cache, item->block_order);
if (real_size_byte == item->real_size_byte && if (real_size_byte == item->real_size_byte &&
announced_size_byte == item->fake_size_byte && announced_size_byte == item->fake_size_byte &&
wrap == item->wrap && wrap == item->wrap &&
item_cache_byte == (cache_size_block << block_order) &&
!need_reset &&
block_order == item->block_order) { block_order == item->block_order) {
success++; success++;
printf("\t\tPerfect!\tMax # of probed blocks: %i\n\n", printf("\t\tPerfect!\tMax # of probed blocks: %i\n\n",
@ -292,12 +298,16 @@ static int unit_test(const char *filename)
} else { } else {
double ret_f_real = real_size_byte; double ret_f_real = real_size_byte;
double ret_f_fake = announced_size_byte; double ret_f_fake = announced_size_byte;
double ret_f_cache = cache_size_block << block_order;
const char *ret_unit_real = adjust_unit(&ret_f_real); const char *ret_unit_real = adjust_unit(&ret_f_real);
const char *ret_unit_fake = adjust_unit(&ret_f_fake); const char *ret_unit_fake = adjust_unit(&ret_f_fake);
printf("\tError\t%s/%.2f %s/%.2f %s/2^%i Byte/2^%i Byte\n\n", const char *ret_unit_cache = adjust_unit(&f_cache);
printf("\tError\t%s/%.2f %s/%.2f %s/2^%i Byte/%.2f %s/%s/2^%i Byte\n\n",
fake_type_to_name(fake_type), fake_type_to_name(fake_type),
ret_f_real, ret_unit_real, ret_f_real, ret_unit_real,
ret_f_fake, ret_unit_fake, wrap, block_order); ret_f_fake, ret_unit_fake, wrap,
ret_f_cache, ret_unit_cache,
need_reset ? "yes" : "no", block_order);
} }
} }
@ -325,6 +335,16 @@ static void report_order(const char *prefix, int order)
printf("%s %.2f %s (2^%i Bytes)\n", prefix, f, unit, order); printf("%s %.2f %s (2^%i Bytes)\n", prefix, f, unit, order);
} }
static void report_cache(const char *prefix, uint64_t cache_size_block,
int need_reset, int order)
{
double f = (cache_size_block << order);
const char *unit = adjust_unit(&f);
printf("%s %.2f %s (%" PRIu64 " blocks), need-reset=%s\n",
prefix, f, unit, cache_size_block,
need_reset ? "yes" : "no");
}
static void report_ops(const char *op, uint64_t count, uint64_t time_us) static void report_ops(const char *op, uint64_t count, uint64_t time_us)
{ {
printf("Probe %s op: count=%" PRIu64 printf("Probe %s op: count=%" PRIu64
@ -338,8 +358,8 @@ static int test_device(struct args *args)
double time_s; double time_s;
struct device *dev, *pdev; struct device *dev, *pdev;
enum fake_type fake_type; enum fake_type fake_type;
uint64_t real_size_byte, announced_size_byte; uint64_t real_size_byte, announced_size_byte, cache_size_block;
int wrap, block_order; int wrap, need_reset, block_order;
uint64_t read_count, read_time_us; uint64_t read_count, read_time_us;
uint64_t write_count, write_time_us; uint64_t write_count, write_time_us;
uint64_t reset_count, reset_time_us; uint64_t reset_count, reset_time_us;
@ -383,7 +403,7 @@ static int test_device(struct args *args)
* the state of the drive. * the state of the drive.
*/ */
assert(!probe_device(dev, &real_size_byte, &announced_size_byte, assert(!probe_device(dev, &real_size_byte, &announced_size_byte,
&wrap, &block_order)); &wrap, &cache_size_block, &need_reset, &block_order));
assert(!gettimeofday(&t2, NULL)); assert(!gettimeofday(&t2, NULL));
if (!args->debug && args->reset_type == RT_MANUAL_USB) { if (!args->debug && args->reset_type == RT_MANUAL_USB) {
@ -450,11 +470,14 @@ static int test_device(struct args *args)
time_s = (t2.tv_sec - t1.tv_sec) + (t2.tv_usec - t1.tv_usec)/1000000.; time_s = (t2.tv_sec - t1.tv_sec) + (t2.tv_usec - t1.tv_usec)/1000000.;
printf("\nDevice geometry:\n"); printf("\nDevice geometry:\n");
report_size("\t *Usable* size:", real_size_byte, block_order); report_size("\t *Usable* size:", real_size_byte,
report_size("\t Announced size:", announced_size_byte,
block_order); block_order);
report_order("\t Module:", wrap); report_size("\t Announced size:", announced_size_byte,
report_order("\tPhysical block size:", block_order); block_order);
report_order("\t Module:", wrap);
report_cache("\tPermanent cache size:", cache_size_block,
need_reset, block_order);
report_order("\t Physical block size:", block_order);
printf("\nProbe time: %.2f seconds\n", time_s); printf("\nProbe time: %.2f seconds\n", time_s);
if (args->time_ops) { if (args->time_ops) {

7
libdevs.c
View File

@ -918,13 +918,6 @@ static inline struct perf_device *dev_pdev(struct device *dev)
return (struct perf_device *)dev; return (struct perf_device *)dev;
} }
static inline uint64_t diff_timeval_us(const struct timeval *t1,
const struct timeval *t2)
{
return (t2->tv_sec - t1->tv_sec) * 1000000ULL +
t2->tv_usec - t1->tv_usec;
}
static int pdev_read_block(struct device *dev, char *buf, int length, static int pdev_read_block(struct device *dev, char *buf, int length,
uint64_t offset) uint64_t offset)
{ {

809
libprobe.c
View File

@ -4,125 +4,74 @@
#include <assert.h> #include <assert.h>
#include <math.h> #include <math.h>
#include <errno.h> #include <errno.h>
#include <time.h> /* For time(). */
#include <sys/time.h> /* For gettimeofday(). */
#include "libutils.h" #include "libutils.h"
#include "libprobe.h" #include "libprobe.h"
static inline int equal_blk(struct device *dev, const char *b1, const char *b2) static int write_blocks(struct device *dev,
{ uint64_t first_pos, uint64_t last_pos, uint64_t salt)
return !memcmp(b1, b2, dev_get_block_size(dev));
}
/* Return true if @b1 and b2 are at most @tolerance_byte bytes different. */
static int similar_blk(struct device *dev, const char *b1, const char *b2,
int tolerance_byte)
{ {
const int block_order = dev_get_block_order(dev);
const int block_size = dev_get_block_size(dev); const int block_size = dev_get_block_size(dev);
int i; uint64_t offset = first_pos << block_order;
/* Aligning these pointers is necessary to directly read and write
for (i = 0; i < block_size; i++) { * the block device.
if (*b1 != *b2) { * For the file device, this is superfluous.
tolerance_byte--;
if (tolerance_byte <= 0)
return false;
}
b1++;
b2++;
}
return true;
}
/* Return true if the block at @pos is damaged. */
static int test_block(struct device *dev,
const char *stamp_blk, char *probe_blk, uint64_t pos)
{
/* Write block. */
if (dev_write_block(dev, stamp_blk, pos) &&
dev_write_block(dev, stamp_blk, pos))
return true;
/* Reset. */
if (dev_reset(dev) && dev_reset(dev))
return true;
/*
* Test block.
*/ */
char stack[align_head(block_order) + (1 << block_order)];
char *stamp_blk = align_mem(stack, block_order);
uint64_t pos;
if (dev_read_block(dev, probe_blk, pos) && for (pos = first_pos; pos <= last_pos; pos++) {
dev_read_block(dev, probe_blk, pos)) fill_buffer_with_block(stamp_blk, block_order, offset, salt);
return true; if (dev_write_block(dev, stamp_blk, pos) &&
dev_write_block(dev, stamp_blk, pos))
if (equal_blk(dev, stamp_blk, probe_blk))
return false;
/* Save time with certainly damaged blocks. */
if (!similar_blk(dev, stamp_blk, probe_blk, 8)) {
/* The probe block is damaged. */
return true;
}
/* The probe block seems to be damaged.
* Trying a second time...
*/
return dev_write_and_reset(dev, stamp_blk, pos) ||
dev_read_block(dev, probe_blk, pos) ||
!equal_blk(dev, stamp_blk, probe_blk);
}
/* Minimum size of the memory chunk used to build flash drives.
* It must be a power of two.
*/
static inline uint64_t initial_high_bit_block(struct device *dev)
{
int block_order = dev_get_block_order(dev);
assert(block_order <= 20);
return 1ULL << (20 - block_order);
}
/* Caller must guarantee that the left bock is good, and written. */
static int search_wrap(struct device *dev,
uint64_t left_pos, uint64_t *pright_pos,
const char *stamp_blk, char *probe_blk)
{
uint64_t high_bit = initial_high_bit_block(dev);
uint64_t pos = high_bit + left_pos;
/* The left block must be in the first memory chunk. */
assert(left_pos < high_bit);
/* Check that the drive has at least one memory chunk. */
assert((high_bit - 1) <= *pright_pos);
while (pos < *pright_pos) {
if (dev_read_block(dev, probe_blk, pos) &&
dev_read_block(dev, probe_blk, pos))
return true; return true;
/* XXX Deal with flipped bit on reception. */ offset += block_size;
if (equal_blk(dev, stamp_blk, probe_blk)) {
/* XXX Test wraparound hypothesis. */
*pright_pos = high_bit - 1;
return false;
}
high_bit <<= 1;
pos = high_bit + left_pos;
} }
return false; return false;
} }
static int high_level_reset(struct device *dev, uint64_t start_pos,
uint64_t cache_size_block, int need_reset, uint64_t salt)
{
if (write_blocks(dev,
start_pos, start_pos + cache_size_block - 1, salt))
return true;
/* Reset. */
if (need_reset && dev_reset(dev) && dev_reset(dev))
return true;
return false;
}
/* Statistics used by bisect() in order to optimize the proportion
* between writes and resets.
*/
struct bisect_stats {
int write_count;
int reset_count;
uint64_t write_time_us;
uint64_t reset_time_us;
};
static void init_bisect_stats(struct bisect_stats *stats)
{
memset(stats, 0, sizeof(*stats));
}
#define MAX_N_BLOCK_ORDER 10 #define MAX_N_BLOCK_ORDER 10
static uint64_t estimate_best_n_block(struct device *dev) static uint64_t estimate_n_bisect_blocks(struct bisect_stats *pstats)
{ {
uint64_t write_count, write_time_us;
uint64_t reset_count, reset_time_us;
double t_w_us, t_2w_us, t_r_us; double t_w_us, t_2w_us, t_r_us;
uint64_t n_block_order; uint64_t n_block_order;
perf_device_sample(dev, NULL, NULL, &write_count, &write_time_us, if (pstats->write_count < 3 || pstats->reset_count < 1) {
&reset_count, &reset_time_us);
if (write_count < 3 || reset_count < 2) {
/* There is not enough measurements. */ /* There is not enough measurements. */
return (1 << 2) - 1; return (1 << 2) - 1;
} }
@ -179,8 +128,8 @@ static uint64_t estimate_best_n_block(struct device *dev)
* *
* We approximate Tw' making it equal to Tw. * We approximate Tw' making it equal to Tw.
*/ */
t_w_us = (double)write_time_us / write_count; t_w_us = (double)pstats->write_time_us / pstats->write_count;
t_r_us = (double)reset_time_us / reset_count; t_r_us = (double)pstats->reset_time_us / pstats->reset_count;
t_2w_us = t_w_us > 0. ? 2. * t_w_us : 1.; /* Avoid zero division. */ t_2w_us = t_w_us > 0. ? 2. * t_w_us : 1.; /* Avoid zero division. */
n_block_order = ilog2(round(t_r_us / t_2w_us + 3.)); n_block_order = ilog2(round(t_r_us / t_2w_us + 3.));
@ -193,118 +142,497 @@ static uint64_t estimate_best_n_block(struct device *dev)
return (1 << n_block_order) - 1; return (1 << n_block_order) - 1;
} }
/* Write blocks whose offsets are after @left_pos but /* Write blocks whose offsets are after @left_pos and before @right_pos. */
* less or equal to @right_pos. static int write_bisect_blocks(struct device *dev,
*/ uint64_t left_pos, uint64_t right_pos, uint64_t n_blocks,
static int write_test_blocks(struct device *dev, const char *stamp_blk, uint64_t salt, uint64_t *pa, uint64_t *pb, uint64_t *pmax_idx)
uint64_t left_pos, uint64_t right_pos,
uint64_t *pa, uint64_t *pb, uint64_t *pmax_idx)
{ {
uint64_t pos, last_pos; uint64_t pos, last_pos;
uint64_t n_block = estimate_best_n_block(dev);
assert(n_block >= 1); assert(n_blocks >= 1);
/* Find coeficients of function a*idx + b where idx <= max_idx. */ /* Find coeficients of function a*idx + b where idx <= max_idx. */
assert(left_pos < right_pos); assert(left_pos < right_pos);
assert(right_pos - left_pos >= 2);
*pb = left_pos + 1; *pb = left_pos + 1;
*pa = round((right_pos - *pb) / (n_block + 1.)); *pa = round((right_pos - *pb - 1.) / (n_blocks + 1.));
*pa = !*pa ? 1ULL : *pa; *pa = !*pa ? 1ULL : *pa;
*pmax_idx = (right_pos - *pb) / *pa; *pmax_idx = (right_pos - *pb - 1) / *pa;
if (*pmax_idx >= n_block) { if (*pmax_idx >= n_blocks) {
/* Shift the zero of the function to the right. /* Shift the zero of the function to the right.
* This avoids picking the leftmost block when a more * This avoids picking the leftmost block when a more
* informative block to the right is available. * informative block to the right is available.
* This also biases toward righter blocks,
* what improves the time to test good flash drives.
*/ */
*pb += *pa; *pb += *pa;
*pmax_idx = n_block - 1; *pmax_idx = n_blocks - 1;
} }
last_pos = *pa * *pmax_idx + *pb; last_pos = *pa * *pmax_idx + *pb;
assert(last_pos <= right_pos); assert(last_pos < right_pos);
/* Write test blocks. */ /* Write test blocks. */
for (pos = *pb; pos <= last_pos; pos += *pa) for (pos = *pb; pos <= last_pos; pos += *pa)
if (dev_write_block(dev, stamp_blk, pos) && if (write_blocks(dev, pos, pos, salt))
dev_write_block(dev, stamp_blk, pos))
return true; return true;
return false; return false;
} }
/* Return true if the test block at @pos is damaged. */ static int is_block_good(struct device *dev, uint64_t pos, int *pis_good,
static int test_test_block(struct device *dev, uint64_t salt)
const char *stamp_blk, char *probe_blk, uint64_t pos)
{ {
const int block_order = dev_get_block_order(dev);
char stack[align_head(block_order) + (1 << block_order)];
char *probe_blk = align_mem(stack, block_order);
uint64_t found_offset;
if (dev_read_block(dev, probe_blk, pos) && if (dev_read_block(dev, probe_blk, pos) &&
dev_read_block(dev, probe_blk, pos)) dev_read_block(dev, probe_blk, pos))
return true; return true;
return !equal_blk(dev, stamp_blk, probe_blk); *pis_good = !validate_buffer_with_block(probe_blk, block_order,
&found_offset, salt) &&
found_offset == (pos << block_order);
return false;
} }
static int probe_test_blocks(struct device *dev, static int probe_bisect_blocks(struct device *dev,
const char *stamp_blk, char *probe_blk, uint64_t *pleft_pos, uint64_t *pright_pos, uint64_t salt,
uint64_t *pleft_pos, uint64_t *pright_pos,
uint64_t a, uint64_t b, uint64_t max_idx) uint64_t a, uint64_t b, uint64_t max_idx)
{ {
/* Signed variables. */ /* Signed variables. */
int64_t left_idx = 0; int64_t left_idx = 0;
int64_t right_idx = max_idx; int64_t right_idx = max_idx;
int64_t idx = right_idx;
while (left_idx <= right_idx) { while (left_idx <= right_idx) {
int64_t idx = (left_idx + right_idx) / 2;
uint64_t pos = a * idx + b; uint64_t pos = a * idx + b;
if (test_test_block(dev, stamp_blk, probe_blk, pos)) { int is_good;
right_idx = idx - 1; if (is_block_good(dev, pos, &is_good, salt))
*pright_pos = pos; return true;
} else { if (is_good) {
left_idx = idx + 1; left_idx = idx + 1;
*pleft_pos = pos; *pleft_pos = pos;
} else {
right_idx = idx - 1;
*pright_pos = pos;
} }
idx = (left_idx + right_idx) / 2;
}
return false;
}
/* Caller must guarantee that the left bock is good, and written. */
static int search_edge(struct device *dev,
uint64_t *pleft_pos, uint64_t right_pos,
const char *stamp_blk, char *probe_blk)
{
uint64_t gap = right_pos - *pleft_pos;
uint64_t prv_gap = gap + 1;
while (prv_gap > gap && gap >= 1) {
uint64_t a, b, max_idx;
if (write_test_blocks(dev, stamp_blk, *pleft_pos, right_pos,
&a, &b, &max_idx))
return true;
/* Reset. */
if (dev_reset(dev) && dev_reset(dev))
return true;
if (probe_test_blocks(dev, stamp_blk, probe_blk,
pleft_pos, &right_pos, a, b, max_idx))
return true;
prv_gap = gap;
gap = right_pos - *pleft_pos;
} }
return false; return false;
} }
/* XXX Write random data to make it harder for fake chips to become "smarter". /* This function assumes that the block at @left_pos is good, and
* There would be a random seed. * that the block at @*pright_pos is bad.
* Buffer cannot be all 0x00 or all 0xFF.
*/ */
static void fill_buffer(char *buf, int len) static int bisect(struct device *dev, struct bisect_stats *pstats,
uint64_t left_pos, uint64_t *pright_pos, uint64_t reset_pos,
uint64_t cache_size_block, int need_reset, uint64_t salt)
{ {
memset(buf, 0xAA, len); uint64_t gap = *pright_pos - left_pos;
struct timeval t1, t2;
assert(*pright_pos > left_pos);
while (gap >= 2) {
uint64_t a, b, max_idx;
uint64_t n_blocks = estimate_n_bisect_blocks(pstats);
assert(!gettimeofday(&t1, NULL));
if (write_bisect_blocks(dev, left_pos, *pright_pos, n_blocks,
salt, &a, &b, &max_idx))
return true;
assert(!gettimeofday(&t2, NULL));
pstats->write_count += max_idx + 1;
pstats->write_time_us += diff_timeval_us(&t1, &t2);
/* Reset. */
assert(!gettimeofday(&t1, NULL));
if (high_level_reset(dev, reset_pos,
cache_size_block, need_reset, salt))
return true;
assert(!gettimeofday(&t2, NULL));
pstats->reset_count++;
pstats->reset_time_us += diff_timeval_us(&t1, &t2);
if (probe_bisect_blocks(dev, &left_pos, pright_pos, salt,
a, b, max_idx))
return true;
gap = *pright_pos - left_pos;
}
assert(gap == 1);
return false;
}
static int count_good_blocks(struct device *dev, uint64_t *pcount,
uint64_t first_pos, uint64_t last_pos, uint64_t salt)
{
uint64_t pos, count = 0;
for (pos = first_pos; pos <= last_pos; pos++) {
int is_good;
if (is_block_good(dev, pos, &is_good, salt))
return true;
if (is_good)
count++;
}
*pcount = count;
return false;
}
static int assess_reset_effect(struct device *dev,
uint64_t *pcache_size_block, int *pneed_reset, int *pdone,
uint64_t first_pos, uint64_t last_pos, uint64_t salt)
{
uint64_t write_target = (last_pos + 1) - first_pos;
uint64_t b4_reset_count_block, after_reset_count_block;
if (count_good_blocks(dev, &b4_reset_count_block,
first_pos, last_pos, salt))
return true;
/* Reset. */
if (dev_reset(dev) && dev_reset(dev))
return true;
if (count_good_blocks(dev, &after_reset_count_block,
first_pos, last_pos, salt))
return true;
if (after_reset_count_block < write_target) {
assert(after_reset_count_block <= b4_reset_count_block);
*pcache_size_block = after_reset_count_block;
*pneed_reset = after_reset_count_block < b4_reset_count_block;
*pdone = true;
return false;
}
*pdone = false;
return false;
}
static inline uint64_t uint64_rand(void)
{
return ((uint64_t)rand() << 32) | rand();
}
static uint64_t uint64_rand_range(uint64_t a, uint64_t b)
{
uint64_t r = uint64_rand();
assert(a <= b);
return a + (r % (b - a + 1));
}
#define N_BLOCK_SAMPLES 64
static int probabilistic_test(struct device *dev,
uint64_t first_pos, uint64_t last_pos, int *pfound_a_bad_block,
uint64_t salt)
{
uint64_t gap;
int i, n, is_linear;
if (first_pos > last_pos)
goto not_found;
/* Let g be the number of good blocks between
* @first_pos and @last_pos including them.
* Let b be the number of bad and overwritten blocks between
* @first_pos and @last_pos including them.
*
* The probability Pr_g of sampling a good block at random between
* @first_pos and @last_pos is Pr_g = g / (g + b), and
* the probability Pr_1b that among k block samples at least
* one block is bad is Pr_1b = 1 - Pr_g^k.
*
* Assuming Pr_g <= 95% and k = 64, Pr_1b >= 96.2%.
* That is, with high probability (i.e. Pr_1b),
* one can find at least a bad block with k samples
* when most blocks are good (Pr_g).
*/
/* Test @samples. */
gap = last_pos - first_pos + 1;
is_linear = gap <= N_BLOCK_SAMPLES;
n = is_linear ? gap : N_BLOCK_SAMPLES;
for (i = 0; i < n; i++) {
uint64_t sample_pos = is_linear
? first_pos + i
: uint64_rand_range(first_pos, last_pos);
int is_good;
if (is_block_good(dev, sample_pos, &is_good, salt))
return true;
if (!is_good) {
/* Found a bad block. */
*pfound_a_bad_block = true;
return false;
}
}
not_found:
*pfound_a_bad_block = false;
return false;
}
static int uint64_cmp(const void *pa, const void *pb)
{
const uint64_t *pia = pa;
const uint64_t *pib = pb;
return *pia - *pib;
}
static int find_a_bad_block(struct device *dev,
uint64_t left_pos, uint64_t *pright_pos, int *found_a_bad_block,
uint64_t reset_pos, uint64_t cache_size_block, int need_reset,
uint64_t salt)
{
/* We need to list all sampled blocks because
* we need a sorted array; read the code to find the why.
* If the sorted array were not needed, one could save the seed
* of the random sequence and repeat the sequence to read the blocks
* after writing them.
*/
uint64_t samples[N_BLOCK_SAMPLES];
uint64_t gap, prv_sample;
int n, i;
if (*pright_pos <= left_pos + 1)
goto not_found;
/* The code below relies on the same analytical result derived
* in probabilistic_test().
*/
/* Fill up @samples. */
gap = *pright_pos - left_pos - 1;
if (gap <= N_BLOCK_SAMPLES) {
n = gap;
for (i = 0; i < n; i++)
samples[i] = left_pos + 1 + i;
} else {
n = N_BLOCK_SAMPLES;
for (i = 0; i < n; i++)
samples[i] = uint64_rand_range(left_pos + 1,
*pright_pos - 1);
}
/* Sort entries of @samples to minimize reads.
* As soon as one finds a bad block, one can stop and ignore
* the remaining blocks because the found bad block is
* the leftmost bad block.
*/
qsort(samples, n, sizeof(uint64_t), uint64_cmp);
/* Write @samples. */
prv_sample = left_pos;
for (i = 0; i < n; i++) {
if (samples[i] == prv_sample)
continue;
prv_sample = samples[i];
if (write_blocks(dev, prv_sample, prv_sample, salt))
return true;
}
/* Reset. */
if (high_level_reset(dev, reset_pos,
cache_size_block, need_reset, salt))
return true;
/* Test @samples. */
prv_sample = left_pos;
for (i = 0; i < n; i++) {
int is_good;
if (samples[i] == prv_sample)
continue;
prv_sample = samples[i];
if (is_block_good(dev, prv_sample, &is_good, salt))
return true;
if (!is_good) {
/* Found the leftmost bad block. */
*pright_pos = prv_sample;
*found_a_bad_block = true;
return false;
}
}
not_found:
*found_a_bad_block = false;
return false;
}
/* Both need to be a power of 2 and larger than, or equal to 2^block_order. */
#define MIN_CACHE_SIZE_BYTE (1ULL << 20)
#define MAX_CACHE_SIZE_BYTE (1ULL << 30)
static int find_cache_size(struct device *dev,
uint64_t left_pos, uint64_t *pright_pos, uint64_t *pcache_size_block,
int *pneed_reset, int *pgood_drive, const uint64_t salt)
{
const int block_order = dev_get_block_order(dev);
uint64_t write_target = MIN_CACHE_SIZE_BYTE >> block_order;
uint64_t final_write_target = MAX_CACHE_SIZE_BYTE >> block_order;
uint64_t first_pos, last_pos, end_pos;
int done;
/*
* Basis
*
* The key difference between the basis and the inductive step is
* the fact that the basis always calls assess_reset_effect().
* This difference is not for correctness, that is, one can remove it,
* and fold the basis into the inductive step.
* However, this difference is an important speedup because many
* fake drives do not have permanent cache.
*/
assert(write_target > 0);
assert(write_target < final_write_target);
last_pos = end_pos = *pright_pos - 1;
/* This convoluted test is needed because
* the variables are unsigned.
* In a simplified form, it tests the following:
* *pright_pos - write_target > left_pos
*/
if (*pright_pos > left_pos + write_target) {
first_pos = *pright_pos - write_target;
} else if (*pright_pos > left_pos + 1) {
/* There's no room to write @write_target blocks,
* so write what's possible.
*/
first_pos = left_pos + 1;
} else {
goto good;
}
if (write_blocks(dev, first_pos, last_pos, salt))
goto bad;
if (assess_reset_effect(dev, pcache_size_block,
pneed_reset, &done, first_pos, end_pos, salt))
goto bad;
if (done) {
*pright_pos = first_pos;
*pgood_drive = false;
return false;
}
/*
* Inductive step
*/
while (write_target < final_write_target) {
int found_a_bad_block;
write_target <<= 1;
last_pos = first_pos - 1;
if (first_pos > left_pos + write_target)
first_pos -= write_target;
else if (first_pos > left_pos + 1)
first_pos = left_pos + 1;
else
break; /* Cannot write any further. */
/* Write @write_target blocks before
* the previously written blocks.
*/
if (write_blocks(dev, first_pos, last_pos, salt))
goto bad;
if (probabilistic_test(dev, first_pos, end_pos,
&found_a_bad_block, salt))
goto bad;
if (found_a_bad_block) {
if (assess_reset_effect(dev, pcache_size_block,
pneed_reset, &done, first_pos, end_pos, salt))
goto bad;
assert(done);
*pright_pos = first_pos;
*pgood_drive = false;
return false;
}
}
good:
*pright_pos = end_pos + 1;
*pcache_size_block = 0;
*pneed_reset = false;
*pgood_drive = true;
return false;
bad:
/* *pright_pos does not change. */
*pcache_size_block = 0;
*pneed_reset = false;
*pgood_drive = false;
return true;
}
static int find_wrap(struct device *dev,
uint64_t left_pos, uint64_t *pright_pos,
uint64_t reset_pos, uint64_t cache_size_block, int need_reset,
uint64_t salt)
{
uint64_t offset, high_bit, pos = left_pos + 1;
int is_good, block_order;
/*
* Basis
*/
/* Make sure that there is at least a good block at the beginning
* of the drive.
*/
if (pos >= *pright_pos)
return false;
if (write_blocks(dev, pos, pos, salt) ||
high_level_reset(dev, reset_pos,
cache_size_block, need_reset, salt) ||
is_block_good(dev, pos, &is_good, salt) ||
!is_good)
return true;
/*
* Inductive step
*/
block_order = dev_get_block_order(dev);
offset = pos << block_order;
high_bit = clp2(pos);
if (high_bit <= pos)
high_bit <<= 1;
pos += high_bit;
while (pos < *pright_pos) {
char stack[align_head(block_order) + (1 << block_order)];
char *probe_blk = align_mem(stack, block_order);
uint64_t found_offset;
if (dev_read_block(dev, probe_blk, pos) &&
dev_read_block(dev, probe_blk, pos))
return true;
if (!validate_buffer_with_block(probe_blk, block_order,
&found_offset, salt) &&
found_offset == offset) {
*pright_pos = high_bit;
return false;
}
high_bit <<= 1;
pos = high_bit + left_pos + 1;
}
return false;
} }
int probe_device_max_blocks(struct device *dev) int probe_device_max_blocks(struct device *dev)
{ {
uint64_t num_blocks = dev_get_size_byte(dev) >> const int block_order = dev_get_block_order(dev);
dev_get_block_order(dev); uint64_t num_blocks = dev_get_size_byte(dev) >> block_order;
int n = ceiling_log2(num_blocks); int n = ceiling_log2(num_blocks);
/* Make sure that there is no overflow in the formula below. /* Make sure that there is no overflow in the formula below.
@ -313,85 +641,126 @@ int probe_device_max_blocks(struct device *dev)
assert(MAX_N_BLOCK_ORDER < sizeof(int) - 10); assert(MAX_N_BLOCK_ORDER < sizeof(int) - 10);
return return
/* search_wrap() */ /* find_cache_size() */
(MAX_CACHE_SIZE_BYTE >> (block_order - 1)) +
/* find_wrap() */
1 + 1 +
/* Search_edge() /* The number below is just an educated guess. */
* 128 * (
* The number of used blocks is (p * w); see comments in /* bisect()
* estimate_best_n_block() for the definition of the variables. *
* * The number of used blocks is (p * w); see comments
* p * w = n/m * (2^m - 1) < n/m * 2^m = n * (2^m / m) * in estimate_n_bisect_blocks() for the definition of
* * the variables.
* Let f(m) be 2^m / m. One can prove that f(m + 1) >= f(m) *
* for all m >= 1. Therefore, the following bound is true. * p * w = n/m * (2^m - 1) < n/m * 2^m = n * (2^m / m)
* *
* p * w < n * f(max_m) * Let f(m) be 2^m / m. One can prove that
*/ * f(m + 1) >= f(m) for all m >= 1.
((n << MAX_N_BLOCK_ORDER) / MAX_N_BLOCK_ORDER); * Therefore, the following bound is true.
*
* p * w < n * f(max_m)
*/
((n << MAX_N_BLOCK_ORDER) / MAX_N_BLOCK_ORDER) +
/* find_a_bad_block() */
N_BLOCK_SAMPLES
);
} }
/* XXX Properly handle read and write errors.
* Review each assert to check if them can be removed.
*/
int probe_device(struct device *dev, uint64_t *preal_size_byte, int probe_device(struct device *dev, uint64_t *preal_size_byte,
uint64_t *pannounced_size_byte, int *pwrap, int *pblock_order) uint64_t *pannounced_size_byte, int *pwrap,
uint64_t *pcache_size_block, int *pneed_reset, int *pblock_order)
{ {
uint64_t dev_size_byte = dev_get_size_byte(dev); const uint64_t dev_size_byte = dev_get_size_byte(dev);
const int block_size = dev_get_block_size(dev);
const int block_order = dev_get_block_order(dev); const int block_order = dev_get_block_order(dev);
char stack[align_head(block_order) + (2 << block_order)]; struct bisect_stats stats;
char *stamp_blk, *probe_blk; uint64_t salt, cache_size_block;
/* XXX Don't write at the very beginning of the card to avoid uint64_t left_pos, right_pos, mid_drive_pos, reset_pos;
* losing the partition table. int need_reset, good_drive, wrap, found_a_bad_block;
* But write at a random locations to make harder for fake chips
* to become "smarter".
* And try a couple of blocks if they keep failing.
*/
uint64_t left_pos = 10;
uint64_t right_pos = (dev_size_byte >> block_order) - 1;
struct device *pdev;
assert(dev_size_byte % block_size == 0); assert(block_order <= 20);
/* @left_pos must point to a good block.
* We just point to the last block of the first 1MB of the card
* because this region is reserved for partition tables.
*
* Given that all writing is confined to the interval
* (@left_pos, @right_pos), we avoid losing the partition table.
*/
left_pos = (1ULL << (20 - block_order)) - 1;
/* @right_pos must point to a bad block.
* We just point to the block after the very last block.
*/
right_pos = dev_size_byte >> block_order;
/* @left_pos cannot be equal to @right_pos since
* @left_pos points to a good block, and @right_pos to a bad block.
*/
assert(left_pos < right_pos); assert(left_pos < right_pos);
pdev = create_perf_device(dev); /* I, Michel Machado, define that any drive with less than
if (!pdev) * this number of blocks is fake.
return -ENOMEM;
/* Aligning these pointers is necessary to directly read and write
* the block device.
* For the file device, this is superfluous.
*/ */
stamp_blk = align_mem(stack, block_order); mid_drive_pos = clp2(right_pos / 2);
probe_blk = stamp_blk + block_size;
fill_buffer(stamp_blk, block_size); assert(left_pos < mid_drive_pos);
assert(mid_drive_pos < right_pos);
/* Make sure that there is at least a good block at the beginning /* This call is needed due to rand(). */
* of the drive. srand(time(NULL));
*/
if (test_block(pdev, stamp_blk, probe_blk, left_pos)) salt = uint64_rand();
if (find_cache_size(dev, mid_drive_pos - 1, &right_pos,
&cache_size_block, &need_reset, &good_drive, salt))
goto bad; goto bad;
assert(mid_drive_pos <= right_pos);
reset_pos = right_pos;
if (search_wrap(pdev, left_pos, &right_pos, stamp_blk, probe_blk)) if (find_wrap(dev, left_pos, &right_pos,
reset_pos, cache_size_block, need_reset, salt))
goto bad; goto bad;
wrap = ceiling_log2(right_pos << block_order);
if (search_edge(pdev, &left_pos, right_pos, stamp_blk, probe_blk)) init_bisect_stats(&stats);
goto bad; if (!good_drive) {
if (mid_drive_pos < right_pos)
right_pos = mid_drive_pos;
if (bisect(dev, &stats, left_pos, &right_pos,
reset_pos, cache_size_block, need_reset, salt))
goto bad;
}
*preal_size_byte = (left_pos + 1) << block_order; do {
*pannounced_size_byte = dev_size_byte; if (find_a_bad_block(dev, left_pos, &right_pos,
*pwrap = ceiling_log2(right_pos << block_order); &found_a_bad_block, reset_pos, cache_size_block,
*pblock_order = block_order; need_reset, salt))
goto bad;
if (found_a_bad_block &&
bisect(dev, &stats, left_pos, &right_pos,
reset_pos, cache_size_block, need_reset, salt))
goto bad;
} while (found_a_bad_block);
if (right_pos == left_pos + 1) {
/* Bad drive. */
right_pos = 0;
}
*preal_size_byte = right_pos << block_order;
*pwrap = wrap;
goto out; goto out;
bad: bad:
*preal_size_byte = 0; *preal_size_byte = 0;
*pannounced_size_byte = dev_size_byte;
*pwrap = ceiling_log2(dev_size_byte); *pwrap = ceiling_log2(dev_size_byte);
*pblock_order = block_order;
out: out:
pdev_detach_and_free(pdev); *pannounced_size_byte = dev_size_byte;
return 0; *pcache_size_block = cache_size_block;
*pneed_reset = need_reset;
*pblock_order = block_order;
return false;
} }

3
libprobe.h
View File

@ -8,6 +8,7 @@
int probe_device_max_blocks(struct device *dev); int probe_device_max_blocks(struct device *dev);
int probe_device(struct device *dev, uint64_t *preal_size_byte, int probe_device(struct device *dev, uint64_t *preal_size_byte,
uint64_t *pannounced_size_byte, int *pwrap, int *block_order); uint64_t *pannounced_size_byte, int *pwrap,
uint64_t *pcache_size_block, int *pneed_reset, int *pblock_order);
#endif /* HEADER_LIBPROBE_H */ #endif /* HEADER_LIBPROBE_H */

3
libutils.c
View File

@ -28,8 +28,7 @@ int ilog2(uint64_t x)
return pop(x) - 1; return pop(x) - 1;
} }
/* Least power of 2 greater than or equal to x. */ uint64_t clp2(uint64_t x)
static uint64_t clp2(uint64_t x)
{ {
x = x - 1; x = x - 1;
x = x | (x >> 1); x = x | (x >> 1);

25
libutils.h
View File

@ -3,12 +3,30 @@
#include <stdint.h> #include <stdint.h>
#include <argp.h> /* For struct argp_state. */ #include <argp.h> /* For struct argp_state. */
#include <sys/time.h> /* For struct timeval. */
int ilog2(uint64_t x); int ilog2(uint64_t x);
/* Least power of 2 greater than or equal to x. */
uint64_t clp2(uint64_t x);
int ceiling_log2(uint64_t x); int ceiling_log2(uint64_t x);
const char *adjust_unit(double *ptr_bytes); const char *adjust_unit(double *ptr_bytes);
/*
* The functions align_head() and align_mem() are used to align pointers.
*
* The following example allocates two block on stack and makes sure that
* the blocks are aligned with the block size.
*
* // The number 2 below means two blocks.
* char stack[align_head(block_order) + (2 << block_order)];
* char *stamp_blk, *probe_blk;
* stamp_blk = align_mem(stack, block_order);
* probe_blk = stamp_blk + block_size;
*/
static inline int align_head(int order) static inline int align_head(int order)
{ {
return (1 << order) - 1; return (1 << order) - 1;
@ -28,4 +46,11 @@ void fill_buffer_with_block(void *buf, int block_order, uint64_t offset,
int validate_buffer_with_block(const void *buf, int block_order, int validate_buffer_with_block(const void *buf, int block_order,
uint64_t *pfound_offset, uint64_t salt); uint64_t *pfound_offset, uint64_t salt);
static inline uint64_t diff_timeval_us(const struct timeval *t1,
const struct timeval *t2)
{
return (t2->tv_sec - t1->tv_sec) * 1000000ULL +
t2->tv_usec - t1->tv_usec;
}
#endif /* HEADER_LIBUTILS_H */ #endif /* HEADER_LIBUTILS_H */