• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号

使用go语言绕过pagecache读写文件

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

有时候,我们希望我们的数据不通过page cache的缓冲直接落盘。go语言里,用参数DIRECT打开文件可以实现这一点要求。

但这样做有一个硬性的要求,就是在读写的时候,对应的数据在内存中的地址一定要满足512对齐,即首地址的2进制形式中后面至少要有9个0结尾,且数据长度为512字节的整数倍,否则读写会失败。

我们用go语言中的切片slice来验证这件事。

首先我们建立一个go语言的切片并随便赋一些值:

buf := make([]byte, 8192)
for i := 0; i < 20; i++ {
	buf[i] = byte(i)
}

我们首先尝试一下正常的读写文件过程,先不使用DIRECT参数绕开page cache。

func writeWithoutAlignmentWithoutDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdb",
		os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

这段代码的运行结果如下:

buffer  0xc42000a2a0
buffer[0]  0xc42005a000
write with buffer  0xc42005a004
write succeed

可以看出,这个切片的地址是0xc42000a2a0,这个切片内数据的首地址是0xc42005a000,这是一个与c语言不同的地方,c语言的数据首地址即为其中数据的首地址,而go语言中,切片的地址和切片内数据首地址是不同的。

我们要写入的数据的首地址是从切片的第5个元素开始,其首地址为0xc42005a004,虽然并没有512对齐,但是由于我们没有尝试绕过page cache,所以根据WriteAt函数的返回值err可以看到,我们的写入操作是成功的。

下面我们尝试一下使用DIRECT参数打开文件,绕过page cache进行写数据操作,其他参数不变。

func writeWithoutAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdc",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

这段代码运行后,我们可以看到如下结果。

buffer  0xc42000a2e0
buffer[0]  0xc42005a000
write with buffer  0xc42005a004
write error  write /dev/sdc: invalid argument

看到了write error,WriteAt函数这次的返回值给出的并不是nil了,我们的写入操作失败了,其返回值返回了一个不可理解的invalid argument(非法参数)。

but我们的参数毫无问题啊!下面我们尝试一下把要写入的数据改为512对齐。

func writeWithAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdd",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[512 : 512+512]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

这段代码运行后,结果如下。

white with alignment and DIRECT:
buffer  0xc42000a340
buffer[0]  0xc42005a000
write with buffer  0xc42005a200
write succeed

我们的写操作成功了!而这段代码与上次未成功的不同之处只有一个,那就是将要写入数据的首地址改成了512对齐。

通过这三段go程序,我们很清晰的验证了绕过page cache写文件的条件。

类似的,下面给出验证绕过page cache读文件也需要512对齐条件的代码。

func readWithoutAlignmentWithoutDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdb", os.O_RDONLY, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithoutAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdc", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdd", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[512 : 512+512]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}

这三个函数的运行结果分如下。

read with buffer  0xc42005a002
read succeed [4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

  

read with buffer  0xc42005a002
read error  read /dev/sdc: invalid argument

  

read with buffer  0xc42005a200
read succeed [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

可以看出,由于最初我们将切片的前20位分别赋值为0-20,其他位赋值默认值为0,第一个写入函数将buf[4:516]写入到/dev/sdb中,第二个写入函数写入失败,第三个写入函数将buf[512 : 512+512]写入到/dev/sdd,所以根据读取结果可以看出,我们的读取函数也是ok的。

最后,给出整段测试程序的完整代码。

package main

import (
	"fmt"
	"os"
	"syscall"
	"unsafe"
)

func main() {
	buf := make([]byte, 8192)
	for i := 0; i < 20; i++ {
		buf[i] = byte(i)
	}
	fmt.Println("----------------------------------------")
	fmt.Println("white without alignment and DIRECT:")
	writeWithoutAlignmentWithoutDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("white without alignment but with DIRECT:")
	writeWithoutAlignmentWithDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("white with alignment and DIRECT:")
	writeWithAlignmentWithDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("read without alignment and DIRECT:")
	readWithoutAlignmentWithoutDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("read without alignment but with DIRECT:")
	readWithoutAlignmentWithDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("read with alignment and DIRECT:")
	readWithAlignmentWithDIRECT(buf)
}

func writeWithoutAlignmentWithoutDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdb",
		os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}
func writeWithoutAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdc",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}
func writeWithAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdd",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[512 : 512+512]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

func readWithoutAlignmentWithoutDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdb", os.O_RDONLY, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithoutAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdc", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdd", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[512 : 512+512]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}

  


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
go的接口,并发和并行,协程,信道,缓冲处理,异常处理发布时间:2022-07-10
下一篇:
Go实现基于WebSocket的弹幕服务发布时间:2022-07-10
热门推荐
热门话题
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap