OS34.【Linux】文件IO (3) 缓冲区的补充和模拟实现fopen、fwrite、fclose函数

#include <stdio.h>
#include <unistd.h>
#include <string.h>
int main() 
{
    const char *fstr = "hello fwrite\n";
    const char *str = "hello write\n";
    printf("hello printf\n");    
    fprintf(stdout, "hello fprintf\n");
    fwrite(fstr, strlen(fstr), 1, stdout);
    write(1, str, strlen(str));
    fork();       
    return 0;
}

现象

运行结果:

为什么打印到显示器和重定向到文件的结果不一样? → 推测和fork有关

可以加sleep函数调试一下:

#include <stdio.h>
#include <unistd.h>
#include <string.h>
int main() 
{
    const char *fstr = "hello fwrite\n";
    const char *str = "hello write\n";
    printf("hello printf\n");    
    sleep(2);
    fprintf(stdout, "hello fprintf\n");
    sleep(2);
    fwrite(fstr, strlen(fstr), 1, stdout);
    sleep(2);
    write(1, str, strlen(str));
    sleep(2);
    fork();  
    sleep(2);     
    return 0;
}

运行结果:

对于向显示器打印,每隔2s打印一串字符

对于output.txt,可以看到先写入hello write,接着在程序退出前刷新了剩下的字符串

分析原因

1. 对于output.txt文件写入,采用全缓冲(遇到\n不在刷新,而是等缓冲区被写满了或者进程退出时才刷新)

write函数是系统调用,并没有语言层面的缓冲区,因此直接刷新到文件

父进程执行到fork()时,创建子进程,之前在OS18.【Linux】进程基础知识(2)文章讲过:

用fork()创建子进程时,父子进程共用同一份代码,数据以写时拷贝的方式各自私有

那么子进程会拷贝一份父进程的缓冲区,接着子进程执行fork()之后的代码: sleep(2)和return 0;

那么父子进程退出时会各自刷新自己的C语言缓冲区(C语言缓冲区父子进程各自私有,但是操作系统内的缓冲区只有1个),显然缓冲区需要刷新两次,会打印两份

2. 对于向显示器打印,采用行缓冲,代码上的字符串均以\n结尾,因此每隔2s就刷新一次

那为什么向显示器打印不像写入文件那样打印两次呢?

调试以下代码:

#include <stdio.h>
#include <unistd.h>
#include <string.h>
int main() 
{
    const char *fstr = "hello fwrite\n";
    const char *str = "hello write\n";
    printf("hello printf\n");    
    fprintf(stdout, "hello fprintf\n");
    fwrite(fstr, strlen(fstr), 1, stdout);
    write(1, str, strlen(str));
    fork();   
    return 0;
}

下断点:

r命令开始运行,触发第一个断点,查看stdout指向的结构体:

执行printf后,再查看stdout指向的结构体:

C语言的缓冲区中的字符串是"hello printf\n"

执行fprintf后,再查看stdout指向的结构体:

发现之前的C语言的缓冲区中的字符串是"hello printf\n"被刷新了,换成"hello fprintf\n"

这就能说明:

虽然父子进程的数据以写时拷贝的方式各自私有,由于\n的刷新,导致最后父子进程的C语言缓冲区是空的,也就没有必要刷新到系统缓冲区了,因此向显示器只打印一份

4.模拟实现简陋的fopen、fwrite、fclose函数

准备

先新建如下空文件:

main.c

mystdio.h //myfopen、myfwrite函数的声明

mystdio.c //myfopen、myfwrite函数的实现

makefile //将main.c和mystdio.c放在一起编译,防止出现链接错误

仿照glibc-2.42的写法,先写_IO_FILE结构体的定义,这里为了简化说明,只含_fileno文件描述符成员变量

struct _IO_FILE 
{
    int _fileno; 
};

再重定义struct _IO_FILE

typedef struct _IO_FILE FILE;

最后仿照C标准的fopen、fwrite、fclose函数写myfopen、myfwrite、myfclose函数的声明

FILE * fopen ( const char * filename, const char * mode );
size_t fwrite ( const void * ptr, size_t size, size_t count, FILE * stream );
int fclose ( FILE * stream );

那么有:

FILE * myfopen ( const char * filename, const char * mode );
size_t myfwrite ( const void * ptr, size_t size, size_t count, FILE * stream );
int myfclose ( FILE * stream );

实现fopen函数

fopen函数有以下几个选项:

"r"	read: Open file for input operations. The file must exist.
"w"	write: Create an empty file for output operations. If a file with the same name already exists, its contents are discarded and the file is treated as a new empty file.
"a"	append: Open file for output at the end of a file. Output operations always write data at the end of the file, expanding it. Repositioning operations (fseek, fsetpos, rewind) are ignored. The file is created if it does not exist.
"r+"	read/update: Open a file for update (both for input and output). The file must exist.
"w+"	write/update: Create an empty file and open it for update (both for input and output). If a file with the same name already exists its contents are discarded and the file is treated as a new empty file.
"a+"	append/update: Open a file for update (both for input and output) with all output operations writing data at the end of the file. Repositioning operations (fseek, fsetpos, rewind) affects the next input operations, but output operations move the position back to the end of file. The file is created if it does not exist.

这里只实现"r"、"w"、"a"选项

可以使用if (strcmp(str1,str2)==0)来判断各个选项,其实可以借鉴glibc-2.42中的写法:

glibc库的处理方式

在glibc-2.42中的include/stdio.h,fopen被定义为_IO_new_fopen:

#   define fopen(fname, mode) _IO_new_fopen (fname, mode)

在libio/iofopen.c中有_IO_new_fopen的实现:

FILE *
_IO_new_fopen (const char *filename, const char *mode)
{
  return __fopen_internal (filename, mode, 1);
}

在libio/iofopen.c中有__fopen_internal的实现:

FILE *
__fopen_internal (const char *filename, const char *mode, int is32)
{
  struct locked_FILE
  {
    struct _IO_FILE_plus fp;
#ifdef _IO_MTSAFE_IO
    _IO_lock_t lock;
#endif
    struct _IO_wide_data wd;
  } *new_f = (struct locked_FILE *) malloc (sizeof (struct locked_FILE));

  if (new_f == NULL)
    return NULL;
#ifdef _IO_MTSAFE_IO
  new_f->fp.file._lock = &new_f->lock;
#endif
  _IO_no_init (&new_f->fp.file, 0, 0, &new_f->wd, &_IO_wfile_jumps);
  _IO_JUMPS (&new_f->fp) = &_IO_file_jumps;
  _IO_new_file_init_internal (&new_f->fp);
  if (_IO_file_fopen ((FILE *) new_f, filename, mode, is32) != NULL)
    return __fopen_maybe_mmap (&new_f->fp.file);

  _IO_un_link (&new_f->fp);
  free (new_f);
  return NULL;
}

__fopen_internal将mode选项传给了_IO_file_fopen

在libio/fileops.c中有_IO_file_fopen的实现:

*注:函数结尾的libc_hidden_ver (_IO_new_file_fopen, _IO_file_fopen)表示_IO_new_file_fopen和_IO_file_fopen是同一个函数

FILE *
_IO_new_file_fopen (FILE *fp, const char *filename, const char *mode,
		    int is32not64)
{
  int oflags = 0, omode;
  int read_write;
  int oprot = 0666;
  int i;
  FILE *result;
  const char *cs;
  const char *last_recognized;

  if (_IO_file_is_open (fp))
    return NULL;
  switch (*mode)
    {
    case 'r':
      omode = O_RDONLY;
      read_write = _IO_NO_WRITES;
      break;
    case 'w':
      omode = O_WRONLY;
      oflags = O_CREAT|O_TRUNC;
      read_write = _IO_NO_READS;
      break;
    case 'a':
      omode = O_WRONLY;
      oflags = O_CREAT|O_APPEND;
      read_write = _IO_NO_READS|_IO_IS_APPENDING;
      break;
    default:
      __set_errno (EINVAL);
      return NULL;
    }
  last_recognized = mode;
  for (i = 1; i < 7; ++i)
    {
      switch (*++mode)
	{
	case '\0':
	case ',':
	  break;
	case '+':
	  omode = O_RDWR;
	  read_write &= _IO_IS_APPENDING;
	  last_recognized = mode;
	  continue;
	case 'x':
	  oflags |= O_EXCL;
	  last_recognized = mode;
	  continue;
	case 'b':
	  last_recognized = mode;
	  continue;
	case 'm':
	  fp->_flags2 |= _IO_FLAGS2_MMAP;
	  continue;
	case 'c':
	  fp->_flags2 |= _IO_FLAGS2_NOTCANCEL;
	  continue;
	case 'e':
	  oflags |= O_CLOEXEC;
	  fp->_flags2 |= _IO_FLAGS2_CLOEXEC;
	  continue;
	default:
	  /* Ignore.  */
	  continue;
	}
      break;
    }

  result = _IO_file_open (fp, filename, omode|oflags, oprot, read_write,
			  is32not64);

  if (result != NULL)
    {
      /* Test whether the mode string specifies the conversion.  */
      cs = strstr (last_recognized + 1, ",ccs=");
      if (cs != NULL)
	{
	  /* Yep.  Load the appropriate conversions and set the orientation
	     to wide.  */
	  struct gconv_fcts fcts;
	  struct _IO_codecvt *cc;
	  char *endp = __strchrnul (cs + 5, ',');
	  char *ccs = malloc (endp - (cs + 5) + 3);

	  if (ccs == NULL)
	    {
	      int malloc_err = errno;  /* Whatever malloc failed with.  */
	      (void) _IO_file_close_it (fp);
	      __set_errno (malloc_err);
	      return NULL;
	    }

	  *((char *) __mempcpy (ccs, cs + 5, endp - (cs + 5))) = '\0';
	  strip (ccs, ccs);

	  if (__wcsmbs_named_conv (&fcts, ccs[2] == '\0'
				   ? upstr (ccs, cs + 5) : ccs) != 0)
	    {
	      /* Something went wrong, we cannot load the conversion modules.
		 This means we cannot proceed since the user explicitly asked
		 for these.  */
	      (void) _IO_file_close_it (fp);
	      free (ccs);
	      __set_errno (EINVAL);
	      return NULL;
	    }

	  free (ccs);

	  assert (fcts.towc_nsteps == 1);
	  assert (fcts.tomb_nsteps == 1);

	  fp->_wide_data->_IO_read_ptr = fp->_wide_data->_IO_read_end;
	  fp->_wide_data->_IO_write_ptr = fp->_wide_data->_IO_write_base;

	  /* Clear the state.  We start all over again.  */
	  memset (&fp->_wide_data->_IO_state, '\0', sizeof (__mbstate_t));
	  memset (&fp->_wide_data->_IO_last_state, '\0', sizeof (__mbstate_t));

	  cc = fp->_codecvt = &fp->_wide_data->_codecvt;

	  cc->__cd_in.step = fcts.towc;

	  cc->__cd_in.step_data.__invocation_counter = 0;
	  cc->__cd_in.step_data.__internal_use = 1;
	  cc->__cd_in.step_data.__flags = __GCONV_IS_LAST;
	  cc->__cd_in.step_data.__statep = &result->_wide_data->_IO_state;

	  cc->__cd_out.step = fcts.tomb;

	  cc->__cd_out.step_data.__invocation_counter = 0;
	  cc->__cd_out.step_data.__internal_use = 1;
	  cc->__cd_out.step_data.__flags = __GCONV_IS_LAST | __GCONV_TRANSLIT;
	  cc->__cd_out.step_data.__statep = &result->_wide_data->_IO_state;

	  /* From now on use the wide character callback functions.  */
	  _IO_JUMPS_FILE_plus (fp) = fp->_wide_data->_wide_vtable;

	  /* Set the mode now.  */
	  result->_mode = 1;
	}
    }

  return result;
}
libc_hidden_ver (_IO_new_file_fopen, _IO_file_fopen)

1.omode表示只读、只写、可读可写选项

2.oflag表示控制其他模式的选项(fopen定义必定会调用open,所以会设置O_RDONLY等标志位)

因此omode | oflag可以作为open系统调用的第二个参数

3.oprot表示文件权限掩码

那么自制的myfopen可以这样写:

先用switch判断选项,如果不是r w a的其中任何一个,返回NULL,接着调用open,如果open调用成功,那么创建FILE对象,填写FILE对象中的文件描述符变量,最后myfopen返回FILE*的指针

FILE * myfopen ( const char * filename, const char * mode )
{
    int oflags = 0, omode;
    int oprot=0666;
    switch (*mode)
    {
    case 'r':
      omode = O_RDONLY;
      break;
    case 'w':
      omode = O_WRONLY;
      oflags = O_CREAT|O_TRUNC;
      break;
    case 'a':
      omode = O_WRONLY;
      oflags = O_CREAT|O_APPEND;
      break;
    default:
      return NULL;
    }
    int fd=open(filename, omode|oflags, oprot);
    if (fd==-1)
      return NULL;
    FILE * fp = malloc(sizeof(FILE));
    if (fp==NULL)
    {
      return NULL;
    }
    fp->_fileno = fd;
    return fp;
}

实现myfwrite函数

myfwrite函数内部调用write系统调用:

ssize_t write(int fd, const void buf[.count], size_t count);

fd从FILE结构体中取得,buf和count从myfwrite的参数中取得,write的第3个参数应该传入期望写入的字节数,那么其值为count*size(表示写入count个size大小的元素)

这里以二进制的形式写入

size_t myfwrite ( const void * ptr, size_t size, size_t count, FILE * stream )
{
  return write(stream->_fileno, ptr, count*size);
}

实现myfclose函数

关闭文件描述符对应的文件即可(需要从FILE结构体中取得)

当然文件描述符必须有效,可以自定义失败的返回值

int myfclose ( FILE * stream )
{
  if (stream->_fileno==-1)
    return -1;
  close(stream->_fileno);
  free(stream);
  return 0;
}

测试代码

#include "mystdio.h"
int main()
{
    const char * str = "teststring";
    const char * filename = "test.txt";
    FILE* fp=myfopen(filename, "w");
    myfwrite(str,strlen(str)-1,1,fp);//不需要写入结尾的\0
    myfclose(fp);
    return 0; 
}

编写makefile

test.out:main.c mystdio.c
    gcc -o $@ $^ -std=c99
.PHONY:clean
    rm -rf test.out

运行结果

5.为自制的FILE结构体添加缓冲区

缓冲区分为输出缓冲区(outbuffer),输入缓冲区(inbuffer这里不讲)

先定义3种刷新方式

#define FIUSH_NOW 1 //立刻刷新
#define FIUSH_LINE 2 //行刷新
#define FIUSH_ALL 4 //全刷新

再为FILE结构体添加自己的缓冲区(模拟C语言的缓冲区)、遍历缓冲区的指针、存储刷新方式的变量

typedef unsigned char byte;
#define buf_size 120
struct _IO_FILE 
{
    int _fileno; 
    byte buffer[buf_size];
    int flush_style;
    int buf_pos;
};

修改myfopen函数

默认行刷新,指针置为0

FILE * fp = malloc(sizeof(FILE));
if (fp==NULL)
{
  return NULL;
}
fp->_fileno = fd;
fp->buf_pos = 0;
fp->flush_style = FIUSH_LINE;//默认是行刷新

修改mywrite函数

1.先将字符串写入缓冲区(为了简单起见,不考虑缓冲区满异常处理局部性问题等情况,使用memcpy)

2.然后记录write系统调用所需的参数start和len

3.移动指针,该指针记录下一次写入缓冲区的位置

memcpy(stream->buffer+stream->buf_pos, (char*)ptr, count*size);
size_t start=stream->buf_pos;
size_t len=count*size;
stream->buf_pos += count*size;

立即刷新即立即调用write系统调用

行刷新看\n,可以用strchr(buffer[buf_pos])来获取当前指针指向的字符

下面判断刷新方式,直接用&位运算:

if (stream->flush_style&FIUSH_NOW)
{
  write(stream->_fileno, stream->buffer,stream->buf_pos);//调用write立刻刷新
  stream->buf_pos=0;
}
else if (stream->flush_style&FIUSH_LINE)
{
  if (((char*)stream->buffer)[stream->buf_pos-1]=='\n')
  {
    write(stream->_fileno, stream->buffer+start, len);
    stream->buf_pos=0;
  }
}
else//stream->flush_style&FIUSH_ALL
{
    write(stream->_fileno, stream->buffer,stream->buf_pos);
    stream->buf_pos=0;
}

注意: 要对stream->buf_pos置为0

修改myfclose函数

当进程退出时,要强制刷新缓冲区

int myfclose ( FILE * stream )
{
  if (stream->_fileno==-1)
    return -1;
  if (stream->buf_pos>0)
  {
    write(stream->_fileno, stream->buffer,stream->buf_pos);
    stream->buf_pos=0;
  }
  close(stream->_fileno);
  free(stream);
  return 0;
}

测试代码

int main()
{
    const char * str1 = "teststring1\n";
    const char * str2 = "teststring2\n";
    const char * filename = "test.txt";
    FILE* fp=myfopen(filename, "w");
    myfwrite(str1,strlen(str1),1,fp);//不需要写入结尾的\0
    myfwrite(str2, strlen(str2),1,fp);//不需要写入结尾的\0
    myfclose(fp);
    return 0; 
}

运行结果:

6.刷新的效率问题

使用代码验证:
文件默认是全缓冲,可以通过fprintf(fp,"%s",buf);+fflush(fp);方式验证行缓冲

采用向两个文件写入随机字符串的方式

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <fcntl.h>
#include <string.h>
char *buf;
char rand_char()
{
    const char table[] = "tuvwxdQRHI23JKLMNabc456STrsWXnopqPYOefgABCDEhijklmUVZ01yzFG789";
    return table[rand() % (strlen(table) - 1)];
}

void without_line_flush(size_t len,FILE* fp)
{
    for (int i=0;i<len;i++)
        buf[i]=rand_char();
    buf[len]='\0';
    fprintf(fp,"%s",buf);
}

void with_line_flush(size_t len,FILE* fp)
{
    for (int i=0;i<len-1;i++)
        buf[i]=rand_char();
    buf[len]='\0';
    fprintf(fp,"%s",buf);
    fflush(fp);
}

int main()
{
    int t=1000;
    FILE* fp1=fopen("without_line_flush.txt","w");
    FILE* fp2=fopen("with_line_flush.txt","w");
    buf = (char*)malloc(120);
    if (!buf) 
    { 
        perror("malloc"); 
        exit(-1); 
    }
    srand((unsigned)time(NULL));
    clock_t begin1=clock();
    while (t--)
    {
        without_line_flush(90,fp1);
    }
    clock_t end1=clock();
    t=1000;
    clock_t begin2=clock();
    while (t--)
    {
        with_line_flush(90,fp2);
        fflush(fp1);
    }
    clock_t end2=clock();
    free(buf);
    fclose(fp1);
    fclose(fp2);
    printf("without_line_flush: %lu\n",end1-begin1);
    printf("with_line_flush: %lu\n",end2-begin2);
    return 0;
}

运行结果: