Friday, December 19, 2014

Troubleshooting of Oracle Dataguard

Troubleshooting of Standby Database

There are many times when we need to troubleshoot dataguard status and we want to know if standby databases are in SYNC or not.

Below are few SQL queries which are very useful to troubleshoot any problem with log archives which needs to be applied to standby database

set pages 200
set feedback off
set heading on
Select ('Database named '||name||' is '||open_mode||' and is in '||database_role||' role') as "DATABASE_INFO" from v$database;


set heading off
select ' ' from dual;
select '#################### RECOVERY_INFO ###################' from dual;
set feedback off
set heading on
col host_name for a20
Set lin 135
Select distinct i.instance_name, i.host_name, r.inst_id, r.recovery_mode from gv$archive_dest_status r, gv$instance I where i.inst_id=r.inst_id and r.recovery_mode <> 'IDEL';


set heading off
set feedback off
select ' ' from dual;
select '#################### GAP_INFO ######################' from dual;
set feedback off
set heading on
select process, client_process, status, thread#, sequence# from gv$managed_standby where thread#<>0 and process like 'MRP%';


set heading off
set feedback off
select ' ' from dual;
select '################### LOG_APPLY_INFO ###################' from dual;
set heading on
set feedback off
Select thread#, max(sequence#), applied from v$archived_log where applied = 'YES' group by thread#, applied;


set heading off
set feedback off
select ' ' from dual;
Set feedback off
Set heading off
select '---------------------------------------------------------------------------------' from dual;
Select 'Please apply the log sequence '||sequence#||' of thread '||thread#||'.' from gv$managed_standby where thread#<>0 and status like 'WAIT%';
Select 'MRP0 is '||status||' successfully' from gv$managed_standby where thread#<>0 and status like 'APPL%';
select '---------------------------------------------------------------------------------' from dual;
select 'Archive log sequence '||sequence#||' is not yet archived on primary side also.' from v$archived_log where thread#=(select thread# from v$managed_standby where status like 'WAIT%') and sequence#=(select sequence# from v$managed_standby where status like 'WAIT%');
select 'Archive log sequence '||sequence#||' is not yet archived on primary side also.' from v$archived_log where thread#=(select thread# from v$managed_standby where status like 'WAIT%') and archived='NO';
select ' ' from dual;
select '-----Storage Info----' from dual;


set lin 140
set pages 200
set heading off
set feedbac off
set echo off

Select 'INFO 1: The standby destination '||name||' is of '||round((Total_mb/1024), 0)||' GB, among this '||round((Free_mb/1024), 0)||' GB is free and '||round(((((Total_mb/1024)-(Free_mb/1024))/(Total_mb/1024))*100), 0)||'% is used.' from v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest');

Select 'INFO 2: Storage if fine.' From v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and free_mb>((Total_mb/100)*8);

Select 'INFO 2: WARNING: Storage has only '||round(100-((((Total_mb/1024)-(Free_mb/1024))/(Total_mb/1024))*100), 0)||'% Percentage free space.' From v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and free_mb<((Total_mb/100)*8);

Select 'INFO 2: CRITICAL: Space need to be freed from storage because standby destination is 100% full.' From v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and total_mb=(total_mb-free_mb);


Set lin 80
Set pages 20000
Set heading off
Set feedback off
Set echo off
!rm create_space_standby.rcv
!rm create_space_standby.sql
select 'NOTE: System is About to create Rman rcv file "create_space_standby.rcv".' from dual;
Spool create_space_standby.rcv
Select 'run {' from dual;
Select 'crosscheck archivelog all;' from dual;
select 'delete noprompt archivelog sequence between '||min(sequence#)||' and '||max(sequence#)||' thread '||thread#||';' from v$archived_log where applied='YES' and deleted='NO' group by thread#, applied, deleted;
select '}' from dual;
select 'exit' from dual;


spool off
set lin 140
select 'Rman will automatically start creating the space if archive destination usage is above 50%.' From v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and free_mb<((total_mb/100)*50);


set heading off
set feedback off
set echo off
alter session set nls_date_format='ddmmyyyy_HHMI';
spool create_space_standby.sql
select '!rman target / cmdfile create_space_standby.rcv log Log_standby_purge_'||sysdate||'.log' From v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and free_mb<((total_mb/100)*50);
spool off
@create_space_standby.sql
select ' To manually create space in Standby destination execute latest RMAN file create_space_standby.rcv' from v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and total_mb=(total_mb-free_mb);
select ' To manually create space in Standby destination execute latest RMAN file create_space_standby.rcv' from v$asm_diskgroup where name=(select trim('+' from value) from v$parameter where name='standby_archive_dest') and free_mb<((Total_mb/100)*8);


set feedback off
set heading on
col name for a10
PROMPT ***** Disk Group Details under ASM control ****
select NAME , TYPE, TOTAL_MB, FREE_MB, round(FREE_MB/TOTAL_MB * 100,1) PCT_FREE,BLOCK_SIZE, SECTOR_SIZe, OFFLINE_DISKS, STATE from v$asm_diskgroup;
set feedback on



###############################
# BACKUP_INFO
###############################
set feedback off
ALTER SESSION SET NLS_DATE_FORMAT='yyyymmdd HH:MI';
Set pages 5000
Set lin 135
Col tag for a10
Col COMPRESSED for a11
Col DEVICE_TYPE for a12
Set feedback off
set heading off
set verify off
Select '-----Tags--------' from dual;
Select distinct tag from V$BACKUP_PIECE_DETAILS where tag like 'ARCH%' or tag like '%LEV1' or tag like '%LEV0' or tag like 'CNTR%' or tag like 'SP%';
Select ' ' from dual;
Select ' ' from dual;
Select '--------PLEASE CHOOSE Tag FROM ABOVE:-------' from dual;
Select ' ' from dual;
set heading on
col Media for a10
col START_TIME for a14
col COMPLETION_TIME for a14
set lin 140
select BS_KEY, DEVICE_TYPE, nvl(media,'---X---') as "Media" , TAG, STATUS status, START_TIME, COMPLETION_TIME, round((ELAPSED_SECONDS/60/60), 0) Hours, round((bytes/1024/1024/1024), 0) as "GB", COMPRESSED, DELETED from V$BACKUP_PIECE_DETAILS where tag=upper('&Tag') and deleted='NO' order by device_type, COMPLETION_TIME, deleted;
set heading off
Select ' ' from dual;
select '-------BACKUP INFO FOR L0, L1, AO BACKUPS-------' from dual;
select ' Last Level0 backup is completed on "'||completion_time||'.' from V$BACKUP_PIECE_DETAILS where BS_KEY=(select max(BS_KEY) from V$BACKUP_PIECE_DETAILS where tag='DB_LEV0');
Select ' ' from dual;
select ' Last Level1 backup is completed on "'||completion_time||'.' from V$BACKUP_PIECE_DETAILS where BS_KEY=(select max(BS_KEY) from V$BACKUP_PIECE_DETAILS where tag='DB_LEV1');
Select ' ' from dual;
select ' Last ARCHIVE Only backup is completed on "'||completion_time||'.' from V$BACKUP_PIECE_DETAILS where BS_KEY=(select max(BS_KEY) from V$BACKUP_PIECE_DETAILS where tag='ARCH_ONLY');
select ' ' from dual;
Select ' ' from dual;
select '>>>>>>>>This information is only for reference purpose.<<<<<<<<' from dual;
Select ' ' from dual;
select '>>>>>>>>FINISH<<<<<<<<<<' from dual;
set heading off
set feedback off
select ' ' from dual;
select '******************************' from dual;

How to change Hostname or IP address on Oracle Server

How to change Hostname or IP address on Oracle Server

Scenario:

You want to change Hostname or IP Address or DNS configuration on server where Oracle is running on ASM.
The steps have been written for an installation that splits the ownership of the “Grid Infrastructure” and the database between a user named ORAGRID and a user named ORADB respectively. Make sure you run below given commands from right user.

Environment:

Operating System – Redhat Enterprise Linux 6.4
Database Version – Oracle 11.2.0.4
ASM – Yes
Databases – 7 DB Instances

Step 1:
Check existing configured resources with Oracle home

[oragrid@litms#### ~]$ crs_stat -t
Name           Type           Target    State     Host
------------------------------------------------------------
ora.ASMDATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....ER.lsnr ora....er.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....2FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....4FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....6FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....8FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....0FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....1FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....2FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora.asm        ora.asm.type   ONLINE    ONLINE    litm...-new
ora.cssd       ora.cssd.type  ONLINE    ONLINE    litm...-new
ora.diskmon    ora....on.type OFFLINE   OFFLINE
ora.evmd       ora.evm.type   ONLINE    ONLINE    litm...-new
ora.ons        ora.ons.type   OFFLINE   OFFLINE
ora.rpst02.db  ora....se.type ONLINE    ONLINE    litm...-new

Step 2:
Before start with the Oracle Restart process you must stop the listener:

[oragrid@litms#### ~]$ lsnrctl stop listener
LSNRCTL for Linux: Version 11.2.0.4.0 - Production on 19-DEC-2014 09:50:57
Copyright (c) 1991, 2013, Oracle.  All rights reserved.
Connecting to (DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=EXTPROC1521)))
The command completed successfully

Step 3:
Confirm if listener is stopped

[oragrid@litms#### ~]$ crs_stat -t
Name           Type           Target    State     Host
------------------------------------------------------------
ora.ASMDATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....ER.lsnr ora....er.type OFFLINE   OFFLINE
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....2FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....4FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....6FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....8FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....0FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....1FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....DATA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora....2FRA.dg ora....up.type ONLINE    ONLINE    litm...-new
ora.asm        ora.asm.type   ONLINE    ONLINE    litm...-new
ora.cssd       ora.cssd.type  ONLINE    ONLINE    litm...-new
ora.diskmon    ora....on.type OFFLINE   OFFLINE
ora.evmd       ora.evm.type   ONLINE    ONLINE    litm...-new
ora.ons        ora.ons.type   OFFLINE   OFFLINE
ora.rpst02.db  ora....se.type ONLINE    ONLINE    litm...-new

Step 4:
Login as ROOT
Set ORACLE_HOME as grid home
Execute below command to remove the existing oracle grid infra configuration.

[root@litms#### ~]# $ORACLE_HOME/perl/bin/perl -I $ORACLE_HOME/perl/lib -I $ORACLE_HOME/crs/install $ORACLE_HOME/crs/install/roothas.pl -deconfig
Using configuration parameter file: /u01/app/grid/product/11.2.0.4/crs/install/crsconfig_params
CRS resources for listeners are still configured
PRKO-2573 : ONS daemon is already stopped.
CRS-2529: Unable to act on 'ora.cssd' because that would require stopping or relocating 'ora.asm', but the force option was not specified
CRS-4000: Command Stop failed, or completed with errors.
CRS-2586: Deletion of a running resource 'ora.cssd' requires the force option
CRS-4000: Command Delete failed, or completed with errors.
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.ASMDATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST04DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST04FRA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST06DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST06FRA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST08DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST08FRA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST10DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST10FRA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST11DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST11FRA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST12DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST12FRA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.rpst02.db' on 'litmsj614-new'
CRS-2677: Stop of 'ora.RPST11FRA.dg' on 'litmsj614-new' succeeded
CRS-2677: Stop of 'ora.RPST11DATA.dg' on 'litmsj614-new' succeeded
CRS-2677: Stop of 'ora.ASMDATA.dg' on 'litmsj614-new' succeeded
CRS-2677: Stop of 'ora.rpst02.db' on 'litmsj614-new' succeeded
CRS-2673: Attempting to stop 'ora.RPST02DATA.dg' on 'litmsj614-new'
CRS-2673: Attempting to stop 'ora.RPST02FRA.dg' on 'litmsj614-new'
CRS-2677: Stop of 'ora.RPST02FRA.dg' on 'litmsj614-new' succeeded
CRS-5017: The resource action "ora.RPST12DATA.dg stop" encountered the following error:
ORA-01013: user requested cancel of current operation
. For details refer to "(:CLSN00108:)" in "/u01/app/grid/product/11.2.0.4/log/litmsj614-new/agent/ohasd/oraagent_oragrid/oraagent_oragrid.log".
CRS-5017: The resource action "ora.RPST10FRA.dg stop" encountered the following error:
ORA-01013: user requested cancel of current operation
. For details refer to "(:CLSN00108:)" in "/u01/app/grid/product/11.2.0.4/log/litmsj614-new/agent/ohasd/oraagent_oragrid/oraagent_oragrid.log".
CRS-5017: The resource action "ora.RPST12FRA.dg stop" encountered the following error:
ORA-01013: user requested cancel of current operation
. For details refer to "(:CLSN00108:)" in "/u01/app/grid/product/11.2.0.4/log/litmsj614-new/agent/ohasd/oraagent_oragrid/oraagent_oragrid.log".
CRS-5022: Stop of resource "ora.RPST10DATA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST10DATA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST10DATA.dg' on 'litmsj614-new'
CRS-5022: Stop of resource "ora.RPST04DATA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST04DATA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST04DATA.dg' on 'litmsj614-new'
CRS-5022: Stop of resource "ora.RPST06DATA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST06DATA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST06DATA.dg' on 'litmsj614-new'
CRS-5022: Stop of resource "ora.RPST04FRA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST04FRA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST04FRA.dg' on 'litmsj614-new'
CRS-5022: Stop of resource "ora.RPST06FRA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST06FRA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST06FRA.dg' on 'litmsj614-new'
CRS-5022: Stop of resource "ora.RPST08FRA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST08FRA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST08FRA.dg' on 'litmsj614-new'
CRS-5022: Stop of resource "ora.RPST08DATA.dg" failed: current state is "UNKNOWN"
CRS-2675: Stop of 'ora.RPST08DATA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST08DATA.dg' on 'litmsj614-new'
CRS-2681: Clean of 'ora.RPST10DATA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST04DATA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST06DATA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST04FRA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST06FRA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST08DATA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST08FRA.dg' on 'litmsj614-new' succeeded
CRS-2675: Stop of 'ora.RPST10FRA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST10FRA.dg' on 'litmsj614-new'
CRS-2675: Stop of 'ora.RPST12FRA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST12FRA.dg' on 'litmsj614-new'
CRS-2675: Stop of 'ora.RPST12DATA.dg' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.RPST12DATA.dg' on 'litmsj614-new'
CRS-2681: Clean of 'ora.RPST12FRA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST12DATA.dg' on 'litmsj614-new' succeeded
CRS-2681: Clean of 'ora.RPST10FRA.dg' on 'litmsj614-new' succeeded
CRS-2677: Stop of 'ora.RPST02DATA.dg' on 'litmsj614-new' succeeded
CRS-2679: Attempting to clean 'ora.RPST02DATA.dg' on 'litmsj614-new'
CRS-2681: Clean of 'ora.RPST02DATA.dg' on 'litmsj614-new' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'litmsj614-new'
CRS-2675: Stop of 'ora.asm' on 'litmsj614-new' failed
CRS-2679: Attempting to clean 'ora.asm' on 'litmsj614-new'
CRS-2681: Clean of 'ora.asm' on 'litmsj614-new' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'litmsj614-new'
CRS-2677: Stop of 'ora.cssd' on 'litmsj614-new' succeeded
CRS-2673: Attempting to stop 'ora.evmd' on 'litmsj614-new'
CRS-2677: Stop of 'ora.evmd' on 'litmsj614-new' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'litmsj614-new' has completed
CRS-4133: Oracle High Availability Services has been stopped.

Can't exec "/usr/bin/lsb_release": No such file or directory at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 542.
Use of uninitialized value $LSB_RELEASE in split at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 547.
Can't exec "/usr/bin/lsb_release": No such file or directory at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 542.
Use of uninitialized value $LSB_RELEASE in split at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 547.
Can't exec "/usr/bin/lsb_release": No such file or directory at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 542.
Use of uninitialized value $LSB_RELEASE in split at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 547.
Can't exec "/usr/bin/lsb_release": No such file or directory at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 542.
Use of uninitialized value $LSB_RELEASE in split at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 547.
Can't exec "/usr/bin/lsb_release": No such file or directory at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 542.
Use of uninitialized value $LSB_RELEASE in split at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 547.
Can't exec "/usr/bin/lsb_release": No such file or directory at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 542.
Use of uninitialized value $LSB_RELEASE in split at /u01/app/grid/product/11.2.0.4/lib/osds_acfslib.pm line 547.
Successfully deconfigured Oracle Restart stack

Step 5:
After removing oracle configuration you can change the hostname of your Server

Edit /etc/sysconfig/network file

NETWORKING=yes
HOSTNAME=<New Hostname>

Edit /etc/hosts file

cat /etc/hosts
127.0.0.1         localhost localhost.localdomain localhost4 localhost4.localdomain4
::1               localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.50.4    <New Hostname>

Edit network adapters IP Address configuration i.e. ifcfg-ethX

[root@### ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
HWADDR="##:##:4C:02:A3:02"
ONBOOT="yes"
IPADDR=<New IP Address>
NETMASK=255.255.255.0
GATEWAY=192.168.50.1

Step 6:
Edit Listener.ora file with new Hostname

Step 7:
Login as ROOT
Set ORACLE_HOME as grid home.
Execute below command to recreate grid infra configuration:

[root@litms####~]# $ORACLE_HOME/perl/bin/perl -I $ORACLE_HOME/perl/lib -I $ORACLE_HOME/crs/install $ORACLE_HOME/crs/install/roothas.pl
Using configuration parameter file: /u01/app/grid/product/11.2.0.4/crs/install/crsconfig_params
LOCAL ADD MODE
Creating OCR keys for user 'oragrid', privgrp 'oinstall'..
Operation successful.
LOCAL ONLY MODE
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
CRS-4664: Node litmsj614 successfully pinned.
Adding Clusterware entries to upstart
litmsj614     2014/12/19 10:51:46     /u01/app/grid/product/11.2.0.4/cdata/litmsj614/backup_20141219_105146.olr
Successfully configured Oracle Grid Infrastructure for a Standalone Server

Step 8:
Add Listener  and start it
[oragrid@litms### ~]$ srvctl add listener
[oragrid@litms#### ~]$ srvctl start listener

Step 9:
Create ASM and add Disks, and mount all diskgroups manually

[oragrid@litms#### disks]$ srvctl add asm -d '/dev/oracleasm/disks/*'
[oragrid@litms#### disks]$ srvctl start asm
[oragrid@litmsj614 disks]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.4.0 Production on Fri Dec 19 10:58:39 2014
Copyright (c) 1982, 2013, Oracle.  All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production
With the Automatic Storage Management option
SQL> alter diskgroup rpst02data mount;
SQL> alter diskgroup rpst04data mount;
SQL> alter diskgroup rpst06data mount;
SQL> alter diskgroup rpst08data mount;
SQL> alter diskgroup rpst10data mount;
SQL> alter diskgroup rpst12data mount;
SQL> alter diskgroup rpst11data mount;
alter diskgroup rpst02fra mount;
alter diskgroup rpst04fra mount;
alter diskgroup rpst06fra mount;
alter diskgroup rpst08fra mount;
alter diskgroup rpst10fra mount;
alter diskgroup rpst12fra mount;
alter diskgroup rpst11fra mount;

Check Status of all mounted diskgroup, like below:

[oragrid@litms#### disks]$ srvctl status diskgroup -g rpst02data
Disk Group rpst02data is running on litmsj614

Step 10:
Configure all databases with SRVCTL (Oracle Restart), Configure and Start your database:

[oragrid@litms#### disks]$ srvctl add database -d RPST02 -o $ORACLE_HOME -n RPST02 -p +rpst02data/rpst02/parameterfile/spfile.279.850066205 -a RPST02DATA,RPST02FRA

[oragrid@litms#### disks]$ srvctl config database -d DB11G
Database unique name: RPST02
Database name: RPST02
Oracle home: /u01/app/grid/product/11.2.0.4
Oracle user: oragrid
Spfile: +rpst02data/rpst02/parameterfile/spfile.279.850066205a
Domain:
Start options: open
Stop options: immediate
Database role: PRIMARY
Management policy: AUTOMATIC
Database instance: RPST02
Disk Groups: RPST02DATA,RPST02FRA
Services:

[oragrid@litms#### disks]$  srvctl start database -d DB11G

Tuesday, December 16, 2014

ORA-27300 ORA-27301 ORA-27302

ORA-27300 ORA-27301 ORA-27302


Error Detail (InAlert.log file):

Sun Nov 16 06:00:02 2014
WARNING: process USER (ospid: 26006) was unable to attach SMR.
Errors in file /u01/app/oracle/diag/rdbms/rpst01/rpst01/trace/rpst01_j002_26006.trc:
ORA-27300: OS system dependent <E5> tg <E4> rd: open utf <F6> Rdes not with the status: 2
ORA-27301: OS error: No such file or directory
ORA-27302: error occurred in: sskgmsmr_7

Error Description

SMR stands for shared memory region and we have this instance monitoring segment which is used to store data for health monitoring. Every process which starts up attaches to this and the data in this segment is visible to whoever has this SMR as part of its address space. This gets called during SGA initialization for creation of the shared memory segment and attaching to it.

- The SMR name is constructed from the Oracle Home Name and the SID
- The file which this data is stored in is $ORACLE_HOME/dbs/hc_<SID>.dat

This file is also used by the EM agent to determine the database status.

Reason:
1.       DB might have been upgraded recently during which some files has been manually deleted.
2.       Few Files in $ORACLE_HOME/dbs location could have been deleted while database was running. 

Solution:


In my case hc_<SID>.dat file was not present due to which error was being logged into log file. Luckily, this file is created during the database startup and hence restarting the database resolved this issue. If the file has been moved recently to another location, relocating the file back to dbs location might also resolve.

Non critical error ORA-48180 caught while writing to trace file

Non critical error ORA-48180 caught while writing to trace file

Database Version: 11.2.0.4 with ASM
Operating System: Red Hat Enterprise Linux Server release 6.4 (Santiago)

Error Description:

In Alert.log file:

Tue Dec 16 12:10:06 2014
Non critical error ORA-48180 caught while writing to trace file "/u01/app/oracle/diag/rdbms/rpst01/rpst01/trace/rpst01_ora_30396.trc"
Error message: Linux-x86_64 Error: 28: No space left on device
Additional information: 1
Writing to the above trace file is disabled for now on...
OS Audit file could not be created; failing after 6 retries

Tue Dec 16 12:10:06 2014
Non critical error ORA-48180 caught while writing to trace file "/u01/app/oracle/diag/rdbms/rpst01/rpst01/trace/rpst01_ora_30401.trc"
Error message: Linux-x86_64 Error: 28: No space left on device
Additional information: 1
Writing to the above trace file is disabled for now on...
OS Audit file could not be created; failing after 6 retries
OS Audit file could not be created; failing after 6 retries
OS Audit file could not be created; failing after 6 retries


There are many blogs which suggests to increase MAX_DUMP_FILE_SIZE but this is not the case.

In my case I checked the size and found it already set to UNLIMITED.

SQL> show parameter MAX_DUMP_FILE_SIZE

NAME                                 TYPE        VALUE
------------------------------------ ----------- ------------------------------
max_dump_file_size                   string      unlimited

I was also getting below error while trying to login into the database with OS authentication.

[oradb@litm##### dbs]$ export ORACLE_SID=rpst01
[oradb@litm##### dbs]$ sqlplus / as sysdba
SQL*Plus: Release 11.2.0.4.0 Production on Tue Dec 16 12:13:40 2014
Copyright (c) 1982, 2013, Oracle.  All rights reserved.
ERROR:
ORA-09925: Unable to create audit trail file
Linux-x86_64 Error: 28: No space left on device
Additional information: 9925
ORA-01075: you are currently logged on

Solution:

1.       Go to audit_file_dest directory.
2.       Remove *.aud files.
3.       If files are too many then you may get below error
[oradb@litm##### adump]$ rm -rf *
-bash: /bin/rm: Argument list too long
4.       To overcome error, execute below command
find . -name "*.aud" -print | xargs rm
5.       Try to login into your database, it should be successful.

6.       Error should also disappear from alert.log file.