Category: DBFS

Install and Configure Fuse package and mount points

[root@grac41 Desktop]# yum install fuse fuse-libs kernel-devel
Setting up Install Process
Package fuse-2.8.3-4.0.2.el6.x86_64 already installed and latest version
Package fuse-libs-2.8.3-4.0.2.el6.x86_64 already installed and latest version
Package kernel-devel-2.6.32-431.20.3.el6.x86_64 already installed and latest version
Nothing to do
--> Fuse kits already installed

Configure Fuse for a 3 node RAC cluster 
On grac41:
# echo user_allow_other > /etc/fuse.conf
# chmod 644 /etc/fuse.conf
# cat /etc/fuse.conf 
user_allow_other
# ls -l /etc/fuse.conf
root@grac41 Desktop]# scp /etc/fuse.conf grac42:/etc/fuse.conf
fuse.conf                                                                                         100%   17     0.0KB/s   00:00    
[root@grac41 Desktop]#  scp /etc/fuse.conf grac43:/etc/fuse.conf
fuse.conf                                                                                         100%   17     0.0KB/s   00:00    
[root@grac41 Desktop]# ssh grac42 ls -l /etc/fuse.conf
-rw-r--r--. 1 root root 17 Jul 24 09:44 /etc/fuse.conf
[root@grac41 Desktop]#  ssh grac43 ls -l /etc/fuse.conf
-rw-r--r--. 1 root root 17 Jul 24 09:45 /etc/fuse.conf

On grac41, grac42 and grac43 
# mkdir -p /u01/oradata/dbfs_direct
# chown oracle:dba /u01/oradata/dbfs_direct
# ls -ld  /u01/oradata/dbfs_direct
drwxr-xr-x. 2 oracle dba 4096 Jul 24 09:49 /u01/oradata/dbfs_direct

Create DBFS Filesystem

SQL> connect dbfs_user/dbfs_user
SQL> @?/rdbms/admin/dbfs_create_filesystem dbfs_ts FS1
No errors.
--------
CREATE STORE:
begin dbms_dbfs_sfs.createFilesystem(store_name => 'FS_FS1', tbl_name =>
'T_FS1', tbl_tbs => 'dbfs_ts', lob_tbs => 'dbfs_ts', do_partition => false,
partition_key => 1, do_compress => false, compression => '', do_dedup => false,
do_encrypt => false); end;
--------
REGISTER STORE:
begin dbms_dbfs_content.registerStore(store_name=> 'FS_FS1', provider_name =>
'sample1', provider_package => 'dbms_dbfs_sfs'); end;
--------
MOUNT STORE:
begin dbms_dbfs_content.mountStore(store_name=>'FS_FS1', store_mount=>'FS1');
end;
--------
CHMOD STORE:
declare m integer; begin m := dbms_fuse.fs_chmod('/FS1', 16895); end;
No errors.

Set the NOCACHE storage option to the LOB segment used by DBFS
SQL> ALTER TABLE DBFS_USER.T_FS1 MODIFY LOB (FILEDATA) (NOCACHE LOGGING);
SQL> SELECT owner , table_name , segment_name , logging , cache FROM dba_lobs WHERE tablespace_name='DBFS_TS';

OWNER                          TABLE_NAME                     SEGMENT_NAME                   LOGGING CACHE
------------------------------ ------------------------------ ------------------------------ ------- ----------
DBFS_USER                      T_FS1                          LOB_SFS$_FST_1                 YES     NO

Configure “fuse” library links on all  nodes (connected as “root” OS user):

On grac41,grac42,grac43
[root@grac41 Desktop]# . /u01/app/oracle/product/11204/racdb/bin/oraenv
ORACLE_SID = [root] ? grac41
The Oracle base has been set to /u01/app/oracle
[root@grac41 Desktop]# sqlplus scott/tiger
--> should work !

Configure shared libraries

# echo "/usr/local/lib" >> /etc/ld.so.conf.d/usr_local_lib.conf
# cd /usr/local/lib
# ln -s $ORACLE_HOME/lib/libclntsh.so.11.1
# ln -s $ORACLE_HOME/lib/libnnz11.so
# locate libfuse.so
/lib64/libfuse.so.2
/lib64/libfuse.so.2.8.3
/usr/local/lib/libfuse.so
# ln -s /lib64/libfuse.so.2 libfuse.so
# ldconfig
# ldconfig -p | grep fuse
    libfuse.so.2 (libc6,x86-64) => /lib64/libfuse.so.2
    libfuse.so (libc6,x86-64) => /usr/local/lib/libfuse.so
# ln -s $ORACLE_HOME/bin/dbfs_client /sbin/mount.dbfs

--> Run above commands on all nodes

Download, edit and test mount-dbfs.sh script

The “mount-dbfs.sh” script is also attached in this associated MOS Document (ID: 1591515.1).

Configure mount-dbfs.sh
[oracle@grac41 DBFS]$ cat  mount-dbfs.sh | egrep 'DBNAME|MOUNT_POINT|DBFS_USER|ORACLE_HOME|DBFS_PASS' | more
DBNAME=grac4
MOUNT_POINT= /u01/oradata/dbfs_direct
DBFS_USER=dbfs_user
ORACLE_HOME=/u01/app/oracle/product/11204/racdb
DBFS_PASSWD=dbfs_user

Copy new mount-dbfs.sh script at the “<Grid Infrastructure _HOME>/crs/script/”
# scp mount-dbfs.sh grac41:/u01/app/11204/grid/crs/script
# scp mount-dbfs.sh grac42:/u01/app/11204/grid/crs/script
# scp mount-dbfs.sh grac43:/u01/app/11204/grid/crs/script

On all nodes change and verify protections:
# cd /u01/app/11204/grid/crs/script
# pwd
/u01/app/11204/grid/crs/script
# chown oracle:dba  mount-dbfs.sh 
# chmod 750 mount-dbfs.sh 
# ls -l mount-dbfs.sh 
-rwxr-x---. 1 oracle dba 11596 Jul 24 11:44 

Mount DBFS Filesystem a first time 
manually run  script /u01/app/11204/grid/crs/script/mount-dbfs.sh  (as oracle OS user or the RDBMS Oracle Home owner) 
on all the nodes to mount the DBFS filesystem for the fist time as user oracle

$  mount-dbfs.sh start  

Potential errors during mount 
 - ./mount-dbfs.sh: line 198: /tmp/.dbfs-passwd.txt.31706: No such file or directory     ( crit
 - ./mount-dbfs.sh status reports DBFS as OFFLINE but OS mount if DBFS filesystemis ok ! ( critical )
var/log/messages 
   fuse: failed to exec fusermount: Permission denied
   A dynamic linking error occurred: (libfuse.so: cannot open shared object file: No such file or directory)

To debug and fix these error please read following article. 

Note before deploying mount-dbfs.sh script test script on all nodes by using following sequence
      $  mount-dbfs.sh status 
      $  mount-dbfs.sh start 
      $  mount-dbfs.sh status   ( you repeat this command as DBFS start may take some time )
      $  mount-dbfs.sh stop
      $  mount-dbfs.sh status

Deploy mount-dbfs.sh as a CW resource and verify resource status

[oracle@grac41 DBFS]$ cat  add-dbfs-resource.sh 
#!/bin/bash 
#
# Script name : add-dbfs-resource.sh 
# $GRID_HOME is set 
# 
ACTION_SCRIPT=$GRID_HOME/crs/script/mount-dbfs.sh 
ls -l $ACTION_SCRIPT
RESNAME=dbfs_mount 
DBNAME=grac4
DBNAMEL=`echo $DBNAME | tr A-Z a-z` 
echo DBNAME: $DBNAMEL
ORACLE_HOME=$GRID_HOME
PATH=$ORACLE_HOME/bin:$PATH 
export PATH ORACLE_HOME 
crsctl add resource $RESNAME  -type local_resource  -attr "ACTION_SCRIPT=$ACTION_SCRIPT, 
 CHECK_INTERVAL=30,RESTART_ATTEMPTS=10, START_DEPENDENCIES='hard(ora.$DBNAMEL.db)pullup(ora.$DBNAMEL.db)',
  STOP_DEPENDENCIES='hard(ora.$DBNAMEL.db)',  SCRIPT_TIMEOUT=30

[grid@grac41 DBFS]$ crsctl status resource dbfs_mount
NAME=dbfs_mount
TYPE=local_resource
TARGET=OFFLINE, OFFLINE, OFFLINE
STATE=OFFLINE, OFFLINE, OFFLINE

[grid@grac41 DBFS]$ crsctl start resource dbfs_mount
CRS-2672: Attempting to start 'dbfs_mount' on 'grac41'
CRS-2672: Attempting to start 'dbfs_mount' on 'grac42'
CRS-2672: Attempting to start 'dbfs_mount' on 'grac43'
CRS-2676: Start of 'dbfs_mount' on 'grac41' succeeded
CRS-2676: Start of 'dbfs_mount' on 'grac43' succeeded
CRS-2676: Start of 'dbfs_mount' on 'grac42' succeeded

grid@grac41 DBFS]$ crsctl status resource dbfs_mount
NAME=dbfs_mount
TYPE=local_resource
TARGET=ONLINE          , ONLINE          , ONLINE
STATE=ONLINE on grac41, ONLINE on grac42, ONLINE on grac43

Test DBFS filesystem with  different users from different cluster nodes
[grid@grac41 DBFS]$ touch  /u01/oradata/dbfs_direct/FS1/INFO_from_grid__at_grac41
[root@grac43 ~]# touch  /u01/oradata/dbfs_direct/FS1/INFO_from_root_at_grac43
[oracle@grac42 script]$ touch  /u01/oradata/dbfs_direct/FS1/INFO_from_oracle_at_grac42
[oracle@grac42 script]$ ls -l  /u01/oradata/dbfs_direct/FS1/INFO*
-rw-r--r--. 1 oracle oinstall 0 Jul 25 11:56 /u01/oradata/dbfs_direct/FS1/INFO_from_oracle_at_grac42
-rw-r--r--. 1 root   root     0 Jul 25 11:54 /u01/oradata/dbfs_direct/FS1/INFO_from_root_at_grac43
-rw-r--r--. 1 grid   oinstall 0 Jul 25 11:58 /u01/oradata/dbfs_direct/FS1/INFO_from_grid__at_grac41

Reference

• DBFS (White Paper/Demo): How To Setup & Configure 11.2.0.X DBFS Filesystems On RAC Cluster Configurations (CRS Managed) On Linux. (Doc ID 1591515.1)
• Configuring DBFS on Oracle Database Machine [ID 1054431.1]
• Dbfs File System became unmounted due to mount-dbfs.sh timeout on Exadata Database Machine (Doc ID 1385927.1)
• NOTE:1320683.1 – How to trace DBFS when any failure happens
• How to clean-up failed DBFS configuration (Doc ID 1312236.1)
• Accessing Dbfs Using Oracle Wallet Fails With Input/output error (Doc ID 1358194.1)
• Cluster Resource DBFS in not starting (Doc ID 1401303.1)
• Dbfs File System became unmounted due to mount-dbfs.sh timeout on Exadata Database Machine(Doc ID 1385927.1)
• Errors when mounting a DBFS store using fstab due to missing libraries (Doc ID 1053008.1)
• http://www.oracle-base.com/articles/11g/dbfs-11gr2.php
• Debugging problems when mounting a  DBFS filesystem

Overview

• Always check /var/log/messages for generic mount problems
• Mount your DBFS filesysystem with  dbfs_client to rule out problems with mount-dbfs.sh script
• before deploying mount-dbfs.sh script test script on all nodes by using following sequence

$  mount-dbfs.sh status
$  mount-dbfs.sh start
$  mount-dbfs.sh status   ( if status OFFLINE repeat this command as DBFS start may take some time )
$  mount-dbfs.sh stop
$  mount-dbfs.sh status

•   Version Overview

Linux grac41.example.com 3.8.13-35.1.2.el6uek.x86_64
CRS 11.2.0.4.3
Fuse RPMs used
[oracle@grac41 DBFS]$ rpm -qa | grep fuse
  fuse-2.8.3-4.0.2.el6.x86_64
fuse-libs-2.8.3-4.0.2.el6.x86_64
gvfs-fuse-1.4.3-16.el6_5.x86_64

Debugging generic DBFS filesystem mount problems

To rule out and generic errors first try to mount your DBFS using dbfs_client :
[oracle@grac41 DBFS]$ echo dbfs_user > pw
[oracle@grac41 DBFS]$ dbfs_client dbfs_user@grac41 -otrace_file=/tmp/dbfs.out -otrace_level=1 -otrace_size=0 /u01/oradata/dbfs_direct <pw &
[1] 17049 

If above mount doesn't work check /var/log/messages
If mount works youn can test mount-dbfs.sh start  

Some typical problem reported in /var/log/messages:
Error 1:
Jul 25 10:15:50 grac42 DBFS_/u01/oradata/dbfs_direct: mount-dbfs.sh mounting DBFS at /u01/oradata/dbfs_direct from database grac4
Jul 25 10:15:51 grac42 DBFS_/u01/oradata/dbfs_direct: ORACLE_SID is grac42
Jul 25 10:15:51 grac42 DBFS_/u01/oradata/dbfs_direct: spawning dbfs_client command using SID grac42
Jul 25 10:15:51 grac42 kernel: fuse init (API version 7.20)
Jul 25 10:15:51 grac42 DBFS_/u01/oradata/dbfs_direct: fuse: failed to exec fusermount: Permission denied

Solution : Set proper protection for /bin/fusermount 
# chmod +x /bin/fusermount   
For details read :  DBFS resource not starting as crs resource (Doc ID 1908868.1)

Error 2:
Jul 25 09:26:02 grac43 DBFS_/u01/oradata/dbfs_direct: spawning dbfs_client command using SID grac43
Jul 25 09:26:02 grac43 DBFS_/u01/oradata/dbfs_direct: Fail to load library libfuse.so.
Jul 25 09:26:02 grac43 DBFS_/u01/oradata/dbfs_direct: A dynamic linking error occurred: (libfuse.so: cannot open shared object file: No such file or directory)
Jul 25 09:26:09 grac43 DBFS_/u01/oradata/dbfs_direct: Start -- OFFLINE

Fix 
Check your current Shared Lib config according to Fuse libs
[root@grac42 lib]# ldconfig -p | grep fuse
    libfuse.so.2 (libc6,x86-64) => /lib64/libfuse.so.2
--> Here we are missing libfuse.so

# cd /usr/local/lib
# locate libfuse.so
/lib64/libfuse.so.2
/lib64/libfuse.so.2.8.3
/usr/local/lib/libfuse.so
# ln -s /lib64/libfuse.so.2 libfuse.so
# ldconfig
# ldconfig -p | grep fuse
    libfuse.so.2 (libc6,x86-64) => /lib64/libfuse.so.2
    libfuse.so (libc6,x86-64) => /usr/local/lib/libfuse.so

 

Debugging and fixing  mount-dbfs.sh script before deploying script as a CW resource

Before deploying mount-dbfs.sh as a CW resource you should test on every node that 
  mount-dbfs.sh status
  mount-dbfs.sh start
  mount-dbfs.sh status 
  mount-dbfs.sh stop
  mount-dbfs.sh status 
works on each node and returns the expected results for  mount-dbfs.sh status .

Let's start with the initial mount test 
Error 1: Problem with password file - OS mount fails ( critical ) 
[oracle@grac41 DBFS]$  mount-dbfs.sh start
mount-dbfs.sh mounting DBFS at /u01/oradata/dbfs_direct from database grac4
ORACLE_SID is grac41
spawning dbfs_client command using SID grac41
./mount-dbfs.sh: line 198: /tmp/.dbfs-passwd.txt.31706: No such file or directory
Start -- OFFLINE
--> DBFS FS not mounted 

Checking code aroun line 198:
    (nohup $DBFS_CLIENT ${DBFS_USER}@ -o $MOUNT_OPTIONS \
          $MOUNT_POINT < $DBFS_PWDFILE | $LOGGER -p ${LOGGER_FACILITY}.info 2>&1 & ) &

    $RMF $DBFS_PWDFILE
--> Seems password file was delete - Note nohup .. & runs the process in the background 
Proposed Fix : add a short sleep between starting the client and deleting the password 
    (nohup $DBFS_CLIENT ${DBFS_USER}@ -o $MOUNT_OPTIONS \
          $MOUNT_POINT < $DBFS_PWDFILE | $LOGGER -p ${LOGGER_FACILITY}.info 2>&1 & ) &
    sleep 2   <-- proposed code change 
    $RMF $DBFS_PWDFILE

This code change fixes the error:  /tmp/.dbfs-passwd.txt.31706: No such file or directory was fixed 

Retry the mount  
[oracle@grac41 DBFS]$ mount-dbfs.sh  start
mount-dbfs.sh mounting DBFS at /u01/oradata/dbfs_direct from database grac4
ORACLE_SID is grac41
spawning dbfs_client command using SID grac41
nohup: redirecting stderr to stdout
Start -- OFFLINE
[oracle@grac41 DBFS]$ mount
dbfs-dbfs_user@grac41:/ on /u01/oradata/dbfs_direct type fuse (rw,nosuid,nodev,max_read=1048576,default_permissions,user=oracle)
--> Now DBFS is mounted but the status return wrong 

Error 2: Debugging wrong status - DBFS is OFFLINE but OS mount status is ok ! ( critical )
[oracle@grac41 DBFS]$ mount-dbfs.sh status
Checking status now
Check -- OFFLINE
Note : status remains OFFLINE even OS mount was successful. Even rerunning script doesn't help. 
--> Checking mount-dbfs.sh script 
'check'|'status')
  ### check to see if it is mounted
  ### fire off a short process in perl to do the check (need the alarm builtin)
  logit debug "Checking status now"
  $PERL <<'TOT'
    $timeout = $ENV{'PERL_ALARM_TIMEOUT'};
    $SIG{ALRM} = sub { 
      ### we have a problem and need to cleanup
      exit 3;
      die "timeout" ;
    };
    alarm $timeout;
    eval {
      $STATUSOUT=`$ENV{'STAT'} -f -c "%T" $ENV{'MOUNT_POINT'} 2>&1 `; 
      chomp($STATUSOUT);
      if ( ( $ENV{'SOLARIS'} == 1 && $STATUSOUT eq 'uvfs' ) ||
           ( $ENV{'LINUX'} == 1   && $STATUSOUT eq 'UNKNOWN (0x65735546)' ) ) {
        ### status is okay
        exit 0;
Using strace to find the command how CW detects the filesystem status 
[oracle@grac41 DBFS]$ strace -f -o mount-dbfs.trc mount-dbfs.sh status
26156 execve("/usr/bin/stat", ["/usr/bin/stat", "-f", "-c", "%T", "/u01/oradata/dbfs_direct"], [/* 35 vars */]) = 0
--> The check DBFS status the perl script runs following command 
[oracle@grac41 DBFS]$ /usr/bin/stat -f -c %T /u01/oradata/dbfs_direct
fuseblk
--> From a mounted DBFS filesystem we get returned fuseblk
Changing Line
  ( $ENV{'LINUX'} == 1   && $STATUSOUT eq 'UNKNOWN (0x65735546)' ) ) {
to 
  ( $ENV{'LINUX'} == 1   && $STATUSOUT eq 'fuseblk' ) ) {
Now we get correct status 
[oracle@grac41 DBFS]$  mount-dbfs.sh status
Checking status now
Check -- ONLINE

Error 3 : mount-dbfs.sh start  still report status offline after start ( not critical )
[oracle@grac41 DBFS]$ mount-dbfs.sh start
mount-dbfs.sh mounting DBFS at /u01/oradata/dbfs_direct from database grac4
ORACLE_SID is grac41
spawning dbfs_client command using SID grac41
nohup: redirecting stderr to stdout
Start -- OFFLINE

After waiting some seconds the status report looks good
[oracle@grac41 DBFS]$ mount-dbfs.sh status
Checking status now
Check -- ONLINE
Potential Fix: Increase sleeptime before checking mount status
  ### allow time for the mount table update before checking it
  $SLEEP 1
  ### set return code based on success of mounting
  $SCRIPTPATH status > /dev/null 2>&1
  if [ $? -eq 0 ]; then
    logit info "Start -- ONLINE"
    exit 0
  else
    logit info "Start -- OFFLINE"
    exit 1
Change line 210 from
   $SLEEP 1
to
   $SLEEP 5
Note: This error is not critical as CW will test resource status again and again.

Testing CW resource script : mount-dbfs.sh

[oracle@grac41 DBFS]$ mount-dbfs.sh status
Checking status now
Check -- OFFLINE

[oracle@grac41 DBFS]$ mount-dbfs.sh start
mount-dbfs.sh mounting DBFS at /u01/oradata/dbfs_direct from database grac4
ORACLE_SID is grac41
spawning dbfs_client command using SID grac41
nohup: redirecting stderr to stdout
Start -- ONLINE

[oracle@grac41 DBFS]$ mount-dbfs.sh status
Checking status now
Check -- ONLINE
 --> If status is OFFline repeat the mount-dbfs.sh status  for at least 1 minute .

[oracle@grac41 DBFS]$ mount-dbfs.sh stop
unmounting DBFS from /u01/oradata/dbfs_direct
umounting the filesystem using '/bin/fusermount -u /u01/oradata/dbfs_direct'
Stop - stopped, now not mounted

[oracle@grac41 DBFS]$ mount-dbfs.sh status
Checking status now
Check -- OFFLINE

Manually mount DBFS using dbfs_client

[oracle@grac41 DBFS]$ echo dbfs_user > pw
[oracle@grac41 DBFS]$ dbfs_client dbfs_user@grac41 -otrace_file=/tmp/dbfs.out -otrace_level=1 -otrace_size=0 /u01/oradata/dbfs_direct <pw &
[1] 17049

grid@grac41 ~]$ mount
dbfs-dbfs_user@grac41:/ on /u01/oradata/dbfs_direct type fuse (rw,nosuid,nodev,max_read=1048576,default_permissions,user=oracle)

Test file access 
[oracle@grac41 DBFS]$ touch /u01/oradata/dbfs_direct/FS1/t
[oracle@grac41 DBFS]$ ls  /u01/oradata/dbfs_direct/FS1/t
/u01/oradata/dbfs_direct/FS1/t

Reference

• DBFS resource not starting as crs resource (Doc ID 1908868.1)