第三节 Open()实现分析

Open()方法主要用来创建一个boltdb的DB对象，底层会执行新建或者打开存储数据的文件，当指定的文件不存在时， boltdb就会新建一个数据文件。否则的话，就直接加载指定的数据库文件内容。

值的注意是，boltdb会根据Open时，options传递的参数来判断到底加互斥锁还是共享锁。

新建时： 会调用init()方法，内部主要是新建一个文件，然后第0页、第1页写入元数据信息；第2页写入freelist信息；第3页写入bucket leaf信息。并最终刷盘。

加载时： 会读取第0页内容，也就是元信息。然后对其进行校验和校验，当校验通过后获取pageSize。否则的话，读取操作系统默认的pagesize(一般4k)

上述操作完成后，会通过mmap来映射数据。最后再根据磁盘页中的freelist数据初始化db的freelist字段。

  1. // Open creates and opens a database at the given path.
  2. // If the file does not exist then it will be created automatically.
  3. // Passing in nil options will cause Bolt to open the database with the default options.
  4. func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
  5.     var db = &DB{opened: true}
  7.     // Set default options if no options are provided.
  8.     if options == nil {
  9.         options = DefaultOptions
  10.     }
  11.     db.NoGrowSync = options.NoGrowSync
  12.     db.MmapFlags = options.MmapFlags
  14.     // Set default values for later DB operations.
  15.     db.MaxBatchSize = DefaultMaxBatchSize
  16.     db.MaxBatchDelay = DefaultMaxBatchDelay
  17.     db.AllocSize = DefaultAllocSize
  19.     flag := os.O_RDWR
  20.     if options.ReadOnly {
  21.         flag = os.O_RDONLY
  22.         db.readOnly = true
  23.     }
  25.     // Open data file and separate sync handler for metadata writes.
  26.     db.path = path
  27.     var err error
  28.     // 打开db文件
  29.     if db.file, err = os.OpenFile(db.path, flag|os.O_CREATE, mode); err != nil {
  30.         _ = db.close()
  31.         return nil, err
  32.     }
  34.     // Lock file so that other processes using Bolt in read-write mode cannot
  35.     // use the database  at the same time. This would cause corruption since
  36.     // the two processes would write meta pages and free pages separately.
  37.     // The database file is locked exclusively (only one process can grab the lock)
  38.     // if !options.ReadOnly.
  39.     // The database file is locked using the shared lock (more than one process may
  40.     // hold a lock at the same time) otherwise (options.ReadOnly is set).
  42.     // 只读加共享锁、否则加互斥锁
  43.     if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil {
  44.         _ = db.close()
  45.         return nil, err
  46.     }
  48.     // Default values for test hooks
  49.     db.ops.writeAt = db.file.WriteAt
  51.     // Initialize the database if it doesn't exist.
  52.     if info, err := db.file.Stat(); err != nil {
  53.         return nil, err
  54.     } else if info.Size() == 0 {
  55.         // Initialize new files with meta pages.
  56.         // 初始化新db文件
  57.         if err := db.init(); err != nil {
  58.             return nil, err
  59.         }
  60.     } else {
  61.         // 不是新文件，读取第一页元数据
  62.         // Read the first meta page to determine the page size.
  63.         // 2^12,正好是4k
  64.         var buf [0x1000]byte
  65.         if _, err := db.file.ReadAt(buf[:], 0); err == nil {
  66.             // 仅仅是读取了pageSize
  67.             m := db.pageInBuffer(buf[:], 0).meta()
  68.             if err := m.validate(); err != nil {
  69.                 // If we can't read the page size, we can assume it's the same
  70.                 // as the OS -- since that's how the page size was chosen in the
  71.                 // first place.
  72.                 //
  73.                 // If the first page is invalid and this OS uses a different
  74.                 // page size than what the database was created with then we
  75.                 // are out of luck and cannot access the database.
  76.                 db.pageSize = os.Getpagesize()
  77.             } else {
  78.                 db.pageSize = int(m.pageSize)
  79.             }
  80.         }
  81.     }
  83.     // Initialize page pool.
  84.     db.pagePool = sync.Pool{
  85.         New: func() interface{} {
  86.             // 4k
  87.             return make([]byte, db.pageSize)
  88.         },
  89.     }
  91.     // Memory map the data file.
  92.     // mmap映射db文件数据到内存
  93.     if err := db.mmap(options.InitialMmapSize); err != nil {
  94.         _ = db.close()
  95.         return nil, err
  96.     }
  98.     // Read in the freelist.
  99.     db.freelist = newFreelist()
  100.     // db.meta().freelist=2
  101.     // 读第二页的数据
  102.     // 然后建立起freelist中
  103.     db.freelist.read(db.page(db.meta().freelist))
  105.     // Mark the database as opened and return.
  106.     return db, nil
  107. }
  110. // init creates a new database file and initializes its meta pages.
  111. func (db *DB) init() error {
  112.     // Set the page size to the OS page size.
  113.     db.pageSize = os.Getpagesize()
  115.     // Create two meta pages on a buffer.
  116.     buf := make([]byte, db.pageSize*4)
  117.     for i := 0; i < 2; i++ {
  118.         p := db.pageInBuffer(buf[:], pgid(i))
  119.         p.id = pgid(i)
  120.         // 第0页和第1页存放元数据
  121.         p.flags = metaPageFlag
  123.         // Initialize the meta page.
  124.         m := p.meta()
  125.         m.magic = magic
  126.         m.version = version
  127.         m.pageSize = uint32(db.pageSize)
  128.         m.freelist = 2
  129.         m.root = bucket{root: 3}
  130.         m.pgid = 4
  131.         m.txid = txid(i)
  132.         m.checksum = m.sum64()
  133.     }
  135.     // Write an empty freelist at page 3.
  136.     // 拿到第2页存放freelist
  137.     p := db.pageInBuffer(buf[:], pgid(2))
  138.     p.id = pgid(2)
  139.     p.flags = freelistPageFlag
  140.     p.count = 0
  142.     // 第三块存放叶子page
  143.     // Write an empty leaf page at page 4.
  144.     p = db.pageInBuffer(buf[:], pgid(3))
  145.     p.id = pgid(3)
  146.     p.flags = leafPageFlag
  147.     p.count = 0
  149.     // Write the buffer to our data file.
  150.     // 写入4页的数据
  151.     if _, err := db.ops.writeAt(buf, 0); err != nil {
  152.         return err
  153.     }
  154.     // 刷盘
  155.     if err := fdatasync(db); err != nil {
  156.         return err
  157.     }
  159.     return nil
  160. }
  162. // page retrieves a page reference from the mmap based on the current page size.
  163. func (db *DB) page(id pgid) *page {
  164.     pos := id * pgid(db.pageSize)
  165.     return (*page)(unsafe.Pointer(&db.data[pos]))
  166. }
  168. // pageInBuffer retrieves a page reference from a given byte array based on the current page size.
  169. func (db *DB) pageInBuffer(b []byte, id pgid) *page {
  170.     return (*page)(unsafe.Pointer(&b[id*pgid(db.pageSize)]))
  171. }
  173. // mmap opens the underlying memory-mapped file and initializes the meta references.
  174. // minsz is the minimum size that the new mmap can be.
  175. func (db *DB) mmap(minsz int) error {
  176.     db.mmaplock.Lock()
  177.     defer db.mmaplock.Unlock()
  179.     info, err := db.file.Stat()
  180.     if err != nil {
  181.         return fmt.Errorf("mmap stat error: %s", err)
  182.     } else if int(info.Size()) < db.pageSize*2 {
  183.         return fmt.Errorf("file size too small")
  184.     }
  186.     // Ensure the size is at least the minimum size.
  187.     var size = int(info.Size())
  188.     if size < minsz {
  189.         size = minsz
  190.     }
  191.     size, err = db.mmapSize(size)
  192.     if err != nil {
  193.         return err
  194.     }
  196.     // Dereference all mmap references before unmapping.
  197.     if db.rwtx != nil {
  198.         db.rwtx.root.dereference()
  199.     }
  201.     // Unmap existing data before continuing.
  202.     if err := db.munmap(); err != nil {
  203.         return err
  204.     }
  206.     // Memory-map the data file as a byte slice.
  207.     if err := mmap(db, size); err != nil {
  208.         return err
  209.     }
  211.     // Save references to the meta pages.
  212.     // 获取元数据信息
  213.     db.meta0 = db.page(0).meta()
  214.     db.meta1 = db.page(1).meta()
  216.     // Validate the meta pages. We only return an error if both meta pages fail
  217.     // validation, since meta0 failing validation means that it wasn't saved
  218.     // properly -- but we can recover using meta1. And vice-versa.
  219.     err0 := db.meta0.validate()
  220.     err1 := db.meta1.validate()
  221.     if err0 != nil && err1 != nil {
  222.         return err0
  223.     }
  225.     return nil
  226. }
  228. // mmapSize determines the appropriate size for the mmap given the current size
  229. // of the database. The minimum size is 32KB and doubles until it reaches 1GB.
  230. // Returns an error if the new mmap size is greater than the max allowed.
  231. func (db *DB) mmapSize(size int) (int, error) {
  232.     // Double the size from 32KB until 1GB.
  233.     for i := uint(15); i <= 30; i++ {
  234.         if size <= 1<<i {
  235.             return 1 << i, nil
  236.         }
  237.     }
  239.     // Verify the requested size is not above the maximum allowed.
  240.     if size > maxMapSize {
  241.         return 0, fmt.Errorf("mmap too large")
  242.     }
  244.     // If larger than 1GB then grow by 1GB at a time.
  245.     sz := int64(size)
  246.     if remainder := sz % int64(maxMmapStep); remainder > 0 {
  247.         sz += int64(maxMmapStep) - remainder
  248.     }
  250.     // Ensure that the mmap size is a multiple of the page size.
  251.     // This should always be true since we're incrementing in MBs.
  252.     pageSize := int64(db.pageSize)
  253.     if (sz % pageSize) != 0 {
  254.         sz = ((sz / pageSize) + 1) * pageSize
  255.     }
  257.     // If we've exceeded the max size then only grow up to the max size.
  258.     if sz > maxMapSize {
  259.         sz = maxMapSize
  260.     }
  262.     return int(sz), nil
  263. }